Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added proper errors handling for config and fixed multiple logger instances being created #226

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
+130 −114
Open

Added proper errors handling for config and fixed multiple logger instances being created #226

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
1a47beb
chore: made logger instance unique per file
rohittp0 Mar 28, 2024
2fe8c9a
chore: updated api logging
rohittp0 Mar 28, 2024
f04d49e
chore: added proper error for missing config
rohittp0 Mar 28, 2024
4b6a3ac
chore: added proper error for missing keys
rohittp0 Mar 28, 2024
49dd666
chore: added separate methods for warning and error
rohittp0 Mar 28, 2024
3bbfb2b
Merge branch 'main' into config-fix
rohittp0 Apr 8, 2024
88c363e
EOF review suggestion
rohittp0 Apr 16, 2024
6b4a581
Merge branch 'main' into config-fix
rohittp0 Apr 16, 2024
b37910d
Update src/logger.py to add EOF
rohittp0 Apr 16, 2024
662dd05
Merge branch 'main' into config-fix
rohittp0 Apr 18, 2024
cb9f6e6
Merge branch 'main' into config-fix
rohittp0 May 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 12 additions & 12 deletions devika.py
View file
Open in desktop
Expand Up @@ -55,7 +55,7 @@ def test_connect(data):


@app.route("/api/data", methods=["GET"])
@route_logger(logger)
@route_logger
def data():
project = manager.get_project_list()
models = LLM().list_models()
Expand Down Expand Up @@ -98,7 +98,7 @@ def handle_message(data):


@app.route("/api/is-agent-active", methods=["POST"])
@route_logger(logger)
@route_logger
def is_agent_active():
data = request.json
project_name = data.get("project_name")
Expand All @@ -107,7 +107,7 @@ def is_agent_active():


@app.route("/api/get-agent-state", methods=["POST"])
@route_logger(logger)
@route_logger
def get_agent_state():
data = request.json
project_name = data.get("project_name")
Expand All @@ -116,14 +116,14 @@ def get_agent_state():


@app.route("/api/get-browser-snapshot", methods=["GET"])
@route_logger(logger)
@route_logger
def browser_snapshot():
snapshot_path = request.args.get("snapshot_path")
return send_file(snapshot_path, as_attachment=True)


@app.route("/api/get-browser-session", methods=["GET"])
@route_logger(logger)
@route_logger
def get_browser_session():
project_name = request.args.get("project_name")
agent_state = AgentState.get_latest_state(project_name)
Expand All @@ -135,7 +135,7 @@ def get_browser_session():


@app.route("/api/get-terminal-session", methods=["GET"])
@route_logger(logger)
@route_logger
def get_terminal_session():
project_name = request.args.get("project_name")
agent_state = AgentState.get_latest_state(project_name)
Expand All @@ -147,7 +147,7 @@ def get_terminal_session():


@app.route("/api/run-code", methods=["POST"])
@route_logger(logger)
@route_logger
def run_code():
data = request.json
project_name = data.get("project_name")
Expand All @@ -157,7 +157,7 @@ def run_code():


@app.route("/api/calculate-tokens", methods=["POST"])
@route_logger(logger)
@route_logger
def calculate_tokens():
data = request.json
prompt = data.get("prompt")
Expand All @@ -166,7 +166,7 @@ def calculate_tokens():


@app.route("/api/token-usage", methods=["GET"])
@route_logger(logger)
@route_logger
def token_usage():
project_name = request.args.get("project_name")
token_count = AgentState.get_latest_token_usage(project_name)
Expand All @@ -180,7 +180,7 @@ def real_time_logs():


@app.route("/api/settings", methods=["POST"])
@route_logger(logger)
@route_logger
def set_settings():
data = request.json
print("Data: ", data)
Expand All @@ -190,12 +190,12 @@ def set_settings():


@app.route("/api/settings", methods=["GET"])
@route_logger(logger)
@route_logger
def get_settings():
configs = config.get_config()
return jsonify({"settings": configs})


if __name__ == "__main__":
logger.info("Devika is up and running!")
socketio.run(app, debug=False, port=1337, host="0.0.0.0")
socketio.run(app, debug=False, port=1337, host="0.0.0.0")
rohittp0 marked this conversation as resolved.
Show resolved Hide resolved
131 changes: 86 additions & 45 deletions src/config.py
View file
Open in desktop
@@ -1,86 +1,127 @@
import toml
import os

import toml
from fastlogging import LogInit
from toml import TomlDecodeError


class Config:
_instance = None
_logger = None

def __new__(cls):
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._instance._load_config()
return cls._instance

def _load_config(self):
# If the config file doesn't exist, copy from the sample
if not os.path.exists("config.toml"):
with open("sample.config.toml", "r") as f_in, open("config.toml", "w") as f_out:
f_out.write(f_in.read())

self.config = toml.load("config.toml")
if cls._instance:
return cls._instance

cls._instance = super().__new__(cls)
cls._logger = LogInit(pathName="logs/core.log", console=True, colors=True)
Copy link
Collaborator

@ARajgor ARajgor Apr 19, 2024 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

log path is fetch from config file. we are saving it in data/logs/

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't fetch from config file here as it needs to handle the case when the config file is missing

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sample.config

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would be better of we can avoid IO here as there won't be any way to handle any errors as not even the logging will be enabled. Like what if the user incorrectly edits the sample.config and there is some mistake.

If you are sure that this won't be a concern I can update the code.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or if the user renamed config.example to config etc

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also why creating new logger? just fetch from logger file and log it. so it will be in one file and simple

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The logger will not be initialised at this point, importing here will cause cycling import


try:
cls._instance.config = toml.load("config.toml")
return cls._instance

except FileNotFoundError as e:
cls._logger.critical(f"Configuration file '{os.path.join(os.getcwd(), e.filename)}' not found")
cls._logger.info(f"Did you forget to create 'config.toml' file at the project root ( {os.getcwd()} )?")
except TomlDecodeError as e:
cls._logger.critical(f"There is something wrong with your 'config.toml' file: {e}")

cls._logger.info("Checkout 'config.example.toml' and https://toml.io/en/ for more information on TOML.")
exit(1)

def try_get(self, *keys):
try:
value = self.config
for key in keys:
value = value[key]

return value, None, None
except KeyError as e:
sample = "Some value"
for key in reversed(keys):
sample = {key: sample}

err = f"Key {e} of {'.'.join(keys)} not found in config.toml using 'None' as default value"
fix = f"To fix this, update 'config.toml' with\n\n{toml.dumps(sample)}\n"
return None, err, fix

def try_get_with_warning(self, *keys):
value, err, fix = self.try_get(*keys)
if err:
self._logger.warning(err)
self._logger.info(fix)
return value

def try_get_with_error(self, *keys):
value, err, fix = self.try_get(*keys)
if err:
self._logger.error(err)
self._logger.info(fix)
exit(1)
return value

def get_config(self):
return self.config

def get_bing_api_endpoint(self):
return self.config["API_ENDPOINTS"]["BING"]
return self.try_get_with_error("API_ENDPOINTS", "BING")

def get_bing_api_key(self):
return self.config["API_KEYS"]["BING"]
return self.try_get_with_error("API_KEYS", "BING")

def get_google_search_api_key(self):
return self.config["API_KEYS"]["GOOGLE_SEARCH"]
return self.try_get_with_error("API_KEYS", "GOOGLE_SEARCH")

def get_google_search_engine_id(self):
return self.config["API_KEYS"]["GOOGLE_SEARCH_ENGINE_ID"]
return self.try_get_with_error("API_KEYS", "GOOGLE_SEARCH_ENGINE_ID")

def get_google_search_api_endpoint(self):
return self.config["API_ENDPOINTS"]["GOOGLE"]
return self.try_get_with_error("API_ENDPOINTS", "GOOGLE")

def get_ollama_api_endpoint(self):
return self.config["API_ENDPOINTS"]["OLLAMA"]
return self.try_get_with_error("API_ENDPOINTS", "OLLAMA")

def get_claude_api_key(self):
return self.config["API_KEYS"]["CLAUDE"]
return self.try_get_with_error("API_KEYS", "CLAUDE")

def get_openai_api_key(self):
return self.config["API_KEYS"]["OPENAI"]
return self.try_get_with_error("API_KEYS", "OPENAI")

def get_gemini_api_key(self):
return self.config["API_KEYS"]["GEMINI"]
return self.try_get_with_error("API_KEYS", "GEMINI")

def get_mistral_api_key(self):
return self.config["API_KEYS"]["MISTRAL"]
return self.try_get_with_error("API_KEYS", "MISTRAL")

def get_groq_api_key(self):
return self.config["API_KEYS"]["GROQ"]
return self.try_get_with_error("API_KEYS", "GROQ")

def get_netlify_api_key(self):
return self.config["API_KEYS"]["NETLIFY"]
return self.try_get_with_error("API_KEYS", "NETLIFY")

def get_sqlite_db(self):
return self.config["STORAGE"]["SQLITE_DB"]
return self.try_get_with_error("STORAGE", "SQLITE_DB")

def get_screenshots_dir(self):
return self.config["STORAGE"]["SCREENSHOTS_DIR"]
return self.try_get_with_error("STORAGE", "SCREENSHOTS_DIR")

def get_pdfs_dir(self):
return self.config["STORAGE"]["PDFS_DIR"]
return self.try_get_with_error("STORAGE", "PDFS_DIR")

def get_projects_dir(self):
return self.config["STORAGE"]["PROJECTS_DIR"]
return self.try_get_with_error("STORAGE", "PROJECTS_DIR")

def get_logs_dir(self):
return self.config["STORAGE"]["LOGS_DIR"]
return self.try_get_with_error("STORAGE", "LOGS_DIR")

def get_repos_dir(self):
return self.config["STORAGE"]["REPOS_DIR"]
return self.try_get_with_error("STORAGE", "REPOS_DIR")

def get_logging_rest_api(self):
return self.config["LOGGING"]["LOG_REST_API"] == "true"
return self.try_get_with_warning("LOGGING", "LOG_REST_API") == "true"

def get_logging_prompts(self):
return self.config["LOGGING"]["LOG_PROMPTS"] == "true"
return self.try_get_with_warning("LOGGING", "LOG_PROMPTS") == "true"

def set_bing_api_key(self, key):
self.config["API_KEYS"]["BING"] = key
Expand Down Expand Up @@ -134,24 +175,24 @@ def set_sqlite_db(self, db):
self.config["STORAGE"]["SQLITE_DB"] = db
self.save_config()

def set_screenshots_dir(self, dir):
self.config["STORAGE"]["SCREENSHOTS_DIR"] = dir
def set_screenshots_dir(self, directory):
self.config["STORAGE"]["SCREENSHOTS_DIR"] = directory
self.save_config()

def set_pdfs_dir(self, dir):
self.config["STORAGE"]["PDFS_DIR"] = dir
def set_pdfs_dir(self, directory):
self.config["STORAGE"]["PDFS_DIR"] = directory
self.save_config()

def set_projects_dir(self, dir):
self.config["STORAGE"]["PROJECTS_DIR"] = dir
def set_projects_dir(self, directory):
self.config["STORAGE"]["PROJECTS_DIR"] = directory
self.save_config()

def set_logs_dir(self, dir):
self.config["STORAGE"]["LOGS_DIR"] = dir
def set_logs_dir(self, directory):
self.config["STORAGE"]["LOGS_DIR"] = directory
self.save_config()

def set_repos_dir(self, dir):
self.config["STORAGE"]["REPOS_DIR"] = dir
def set_repos_dir(self, directory):
self.config["STORAGE"]["REPOS_DIR"] = directory
self.save_config()

def set_logging_rest_api(self, value):
Expand Down
66 changes: 29 additions & 37 deletions src/logger.py
View file
Open in desktop
Expand Up @@ -7,10 +7,15 @@


class Logger:
def __init__(self, filename="devika_agent.log"):
config = Config()
logs_dir = config.get_logs_dir()
self.logger = LogInit(pathName=logs_dir + "/" + filename, console=True, colors=True, encoding="utf-8")
_loggers = {}
logger = None

def __new__(cls, filename="devika_agent.log"):
if filename not in cls._loggers:
cls._loggers[filename] = super(Logger, cls).__new__(cls)
cls._loggers[filename].logger = LogInit(pathName=Config().get_logs_dir() + "/" + filename, console=True, colors=True, encoding="utf-8")

return cls._loggers[filename]

def read_log_file(self) -> str:
with open(self.logger.pathName, "r") as file:
Expand All @@ -37,40 +42,27 @@ def exception(self, message: str):
self.logger.flush()


def route_logger(logger: Logger):
"""
Decorator factory that creates a decorator to log route entry and exit points.
The decorator uses the provided logger to log the information.
rohittp0 marked this conversation as resolved.
Show resolved Hide resolved

:param logger: The logger instance to use for logging.
"""

def route_logger(func):
log_enabled = Config().get_logging_rest_api()
logger = Logger()

@wraps(func)
def wrapper(*args, **kwargs):
# Log entry point
if log_enabled:
logger.info(f"{request.path} {request.method}")

def decorator(func):
# Call the actual route function
response = func(*args, **kwargs)

@wraps(func)
def wrapper(*args, **kwargs):
# Log entry point
# Log exit point, including response summary if possible
try:
if log_enabled:
logger.info(f"{request.path} {request.method}")

# Call the actual route function
response = func(*args, **kwargs)

from werkzeug.wrappers import Response

# Log exit point, including response summary if possible
try:
if log_enabled:
if isinstance(response, Response) and response.direct_passthrough:
logger.debug(f"{request.path} {request.method} - Response: File response")
else:
response_summary = response.get_data(as_text=True)
logger.debug(f"{request.path} {request.method} - Response: {response_summary}")
except Exception as e:
logger.exception(f"{request.path} {request.method} - {e})")

return response
return wrapper
return decorator
response_summary = response.get_data(as_text=True)
logger.debug(f"{request.path} {request.method} - Response: {response_summary}")
except Exception as e:
logger.exception(f"{request.path} {request.method} - {e})")

return response
return wrapper

rohittp0 marked this conversation as resolved.
Show resolved Hide resolved