diff --git a/README.md b/README.md index 2d29a4933..97a5875fe 100644 --- a/README.md +++ b/README.md @@ -47,15 +47,25 @@ Scope: Store the data in N stores and test the retrieval with the Rag Test Manag ## Run the level 3 +Make sure you have Docker, Poetry, and Python 3.11 installed and postgres installed. + +Copy the .env.example to .env and fill the variables + + +Start the docker: + ```docker compose up promethai_mem ``` +Use the poetry environment: + ``` poetry shell ``` -Make sure to run +Make sure to run to initialize DB tables ``` python scripts/create_database.py ``` -After that, you can run: +After that, you can run the RAG test manager. + ``` python rag_test_manager.py \ @@ -65,4 +75,4 @@ After that, you can run: --metadata "example_data/metadata.json" ``` - +Examples of metadata structure and test set are in the folder "example_data" diff --git a/level_3/database/database_crud.py b/level_3/database/database_crud.py index b2808dfaf..6d53629c7 100644 --- a/level_3/database/database_crud.py +++ b/level_3/database/database_crud.py @@ -1,10 +1,6 @@ -from contextlib import asynccontextmanager - -import asyncio from contextlib import asynccontextmanager import logging -# from database import AsyncSessionLocal logger = logging.getLogger(__name__) diff --git a/level_3/docker-compose.yml b/level_3/docker-compose.yml index d19ad7c3e..a45bc262c 100644 --- a/level_3/docker-compose.yml +++ b/level_3/docker-compose.yml @@ -26,6 +26,23 @@ services: - promethai_mem_backend ports: - "5432:5432" + superset: + platform: linux/amd64 + build: + context: ./superset + dockerfile: Dockerfile + container_name: superset + environment: + - ADMIN_USERNAME=admin + - ADMIN_EMAIL=vasilije@topoteretes.com + - ADMIN_PASSWORD=admin + - POSTGRES_USER=bla + - POSTGRES_PASSWORD=bla + - POSTGRES_DB=bubu + networks: + - promethai_mem_backend + ports: + - '8088:8088' networks: promethai_mem_backend: name: promethai_mem_backend diff --git a/level_3/rag_test_manager.py b/level_3/rag_test_manager.py index 478adda2d..06b8954cb 100644 --- a/level_3/rag_test_manager.py +++ b/level_3/rag_test_manager.py @@ -29,8 +29,8 @@ import itertools import logging import dotenv dotenv.load_dotenv() -import openai logger = logging.getLogger(__name__) +import openai openai.api_key = os.getenv("OPENAI_API_KEY", "") async def retrieve_latest_test_case(session, user_id, memory_id): diff --git a/level_3/superset/Dockerfile b/level_3/superset/Dockerfile new file mode 100644 index 000000000..2f555da47 --- /dev/null +++ b/level_3/superset/Dockerfile @@ -0,0 +1,24 @@ +FROM apache/superset:latest + +USER root + +RUN pip install mysqlclient + +ENV ADMIN_USERNAME $ADMIN_USERNAME +ENV ADMIN_EMAIL $ADMIN_EMAIL +ENV ADMIN_PASSWORD $ADMIN_PASSWORD + +COPY ./superset-init.sh /superset-init.sh +RUN chmod +x /superset-init.sh + + + + +COPY superset_config.py /app/ +ENV SUPERSET_CONFIG_PATH /app/superset_config.py + +COPY add_database_connections.py /app/ +RUN chmod +x /app/add_database_connections.py + +USER superset +ENTRYPOINT [ "/superset-init.sh" ] \ No newline at end of file diff --git a/level_3/superset/add_database_connections.py b/level_3/superset/add_database_connections.py new file mode 100644 index 000000000..ef18230e0 --- /dev/null +++ b/level_3/superset/add_database_connections.py @@ -0,0 +1,32 @@ +from superset_config import DATABASE_CONNECTIONS +from superset import app +from superset.extensions import db +from superset.models.core import Database + + +def add_database_connections(): + with app.app_context(): + for db_conn in DATABASE_CONNECTIONS: + database_name = db_conn['database_name'] + sqlalchemy_uri = db_conn['sqlalchemy_uri'] + + # Check if database already exists + existing_db = db.session.query(Database).filter_by(database_name=database_name).first() + if existing_db: + print(f"Database {database_name} already exists, skipping") + continue + + # Add new database connection + new_db = Database( + database_name=database_name, + sqlalchemy_uri=sqlalchemy_uri, + ) + db.session.add(new_db) + db.session.commit() + print(f"Added database: {database_name}") + + +if __name__ == "__main__": + add_database_connections() + + diff --git a/level_3/superset/superset-init.sh b/level_3/superset/superset-init.sh new file mode 100755 index 000000000..3ca1e44d7 --- /dev/null +++ b/level_3/superset/superset-init.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -ex + +# create Admin user, you can read these values from env or anywhere else possible +superset fab create-admin --username "$ADMIN_USERNAME" --firstname Superset --lastname Admin --email "$ADMIN_EMAIL" --password "$ADMIN_PASSWORD" + +# Upgrading Superset metastore +superset db upgrade + +# setup roles and permissions +superset superset init + +# Starting server in the background +/bin/sh -c /usr/bin/run-server.sh & + +# Waiting for the server to start +sleep 15 + +## Running the script to add database connections +#python /app/add_database_connections.py + +# Bring the server process back into the foreground so that the script doesn't exit and the server keeps running. +wait $! \ No newline at end of file diff --git a/level_3/superset/superset_config.py b/level_3/superset/superset_config.py new file mode 100755 index 000000000..f5844beba --- /dev/null +++ b/level_3/superset/superset_config.py @@ -0,0 +1,31 @@ +import os + +FEATURE_FLAGS = { + "ENABLE_TEMPLATE_PROCESSING": True, +} + +ENABLE_PROXY_FIX = True +SECRET_KEY = "YOUR_OWN_RANDOM_GENERATED_STRING" # Make sure to generate and use your own secret key + +# PostgreSQL Database credentials +POSTGRES_USER = os.getenv('POSTGRES_USER', 'bla') +POSTGRES_PASSWORD = os.getenv('POSTGRES_PASSWORD', 'bla') +POSTGRES_HOST = os.getenv('POSTGRES_HOST', 'postgres') +POSTGRES_PORT = os.getenv('POSTGRES_PORT', '5432') +POSTGRES_DB = os.getenv('POSTGRES_DB', 'bubu') + +# Constructing the SQLAlchemy PostgreSQL URI +SQLALCHEMY_DATABASE_URI = ( + f'postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@' + f'{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}' +) + +DATABASE_CONNECTIONS = [ + { + 'database_name': 'my_postgres', + 'sqlalchemy_uri': f'postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@' + f'{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}', + }, + + # Add more database connections as needed +] \ No newline at end of file