Added async elements

README.md

@@ -47,15 +47,25 @@ Scope: Store the data in N stores and test the retrieval with the Rag Test Manag
 
 ## Run the level 3 
 
+Make sure you have Docker, Poetry, and Python 3.11 installed and postgres installed.
+
+Copy the .env.example to .env and fill the variables
+
+
+Start the docker:
+
 ```docker compose up promethai_mem   ```
 
+Use the poetry environment:
+
 ``` poetry shell ```
 
-Make sure to run 
+Make sure to run to initialize DB tables
 
 ``` python scripts/create_database.py ```
 
-After that, you can run: 
+After that, you can run the RAG test manager.
+
 
 ``` 
     python rag_test_manager.py \
@@ -65,4 +75,4 @@ After that, you can run:
     --metadata "example_data/metadata.json"
 
 ```
-
+Examples of metadata structure and test set are in the folder "example_data"

level_3/database/database_crud.py

@@ -1,10 +1,6 @@
 
-from contextlib import asynccontextmanager
-
-import asyncio
 from contextlib import asynccontextmanager
 import logging
-# from database import AsyncSessionLocal
 
 logger = logging.getLogger(__name__)
 

level_3/docker-compose.yml

@@ -26,6 +26,23 @@ services:
       - promethai_mem_backend
     ports:
       - "5432:5432"
+  superset:
+    platform: linux/amd64
+    build:
+      context: ./superset
+      dockerfile: Dockerfile
+    container_name: superset
+    environment:
+      - ADMIN_USERNAME=admin
+      - ADMIN_EMAIL=vasilije@topoteretes.com
+      - ADMIN_PASSWORD=admin
+      - POSTGRES_USER=bla
+      - POSTGRES_PASSWORD=bla
+      - POSTGRES_DB=bubu
+    networks:
+      - promethai_mem_backend
+    ports:
+      - '8088:8088'
 networks:
   promethai_mem_backend:
     name: promethai_mem_backend

level_3/rag_test_manager.py

@@ -29,8 +29,8 @@ import itertools
 import logging
 import dotenv
 dotenv.load_dotenv()
-import openai
 logger = logging.getLogger(__name__)
+import openai
 openai.api_key = os.getenv("OPENAI_API_KEY", "")
 
 async def retrieve_latest_test_case(session, user_id, memory_id):

level_3/superset/Dockerfile

@@ -0,0 +1,24 @@
+FROM apache/superset:latest
+
+USER root
+
+RUN pip install mysqlclient
+
+ENV ADMIN_USERNAME $ADMIN_USERNAME
+ENV ADMIN_EMAIL $ADMIN_EMAIL
+ENV ADMIN_PASSWORD $ADMIN_PASSWORD
+
+COPY ./superset-init.sh /superset-init.sh
+RUN chmod +x /superset-init.sh
+
+
+
+
+COPY superset_config.py /app/
+ENV SUPERSET_CONFIG_PATH /app/superset_config.py
+
+COPY add_database_connections.py /app/
+RUN chmod +x /app/add_database_connections.py
+
+USER superset
+ENTRYPOINT [ "/superset-init.sh" ]

level_3/superset/add_database_connections.py

@@ -0,0 +1,32 @@
+from superset_config import DATABASE_CONNECTIONS
+from superset import app
+from superset.extensions import db
+from superset.models.core import Database
+
+
+def add_database_connections():
+    with app.app_context():
+        for db_conn in DATABASE_CONNECTIONS:
+            database_name = db_conn['database_name']
+            sqlalchemy_uri = db_conn['sqlalchemy_uri']
+
+            # Check if database already exists
+            existing_db = db.session.query(Database).filter_by(database_name=database_name).first()
+            if existing_db:
+                print(f"Database {database_name} already exists, skipping")
+                continue
+
+            # Add new database connection
+            new_db = Database(
+                database_name=database_name,
+                sqlalchemy_uri=sqlalchemy_uri,
+            )
+            db.session.add(new_db)
+            db.session.commit()
+            print(f"Added database: {database_name}")
+
+
+if __name__ == "__main__":
+    add_database_connections()
+
+

level_3/superset/superset-init.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -ex
+
+# create Admin user, you can read these values from env or anywhere else possible
+superset fab create-admin --username "$ADMIN_USERNAME" --firstname Superset --lastname Admin --email "$ADMIN_EMAIL" --password "$ADMIN_PASSWORD"
+
+# Upgrading Superset metastore
+superset db upgrade
+
+# setup roles and permissions
+superset superset init
+
+# Starting server in the background
+/bin/sh -c /usr/bin/run-server.sh &
+
+# Waiting for the server to start
+sleep 15
+
+## Running the script to add database connections
+#python /app/add_database_connections.py
+
+# Bring the server process back into the foreground so that the script doesn't exit and the server keeps running.
+wait $!

level_3/superset/superset_config.py

@@ -0,0 +1,31 @@
+import os
+
+FEATURE_FLAGS = {
+    "ENABLE_TEMPLATE_PROCESSING": True,
+}
+
+ENABLE_PROXY_FIX = True
+SECRET_KEY = "YOUR_OWN_RANDOM_GENERATED_STRING"  # Make sure to generate and use your own secret key
+
+# PostgreSQL Database credentials
+POSTGRES_USER = os.getenv('POSTGRES_USER', 'bla')
+POSTGRES_PASSWORD = os.getenv('POSTGRES_PASSWORD', 'bla')
+POSTGRES_HOST = os.getenv('POSTGRES_HOST', 'postgres')
+POSTGRES_PORT = os.getenv('POSTGRES_PORT', '5432')
+POSTGRES_DB = os.getenv('POSTGRES_DB', 'bubu')
+
+# Constructing the SQLAlchemy PostgreSQL URI
+SQLALCHEMY_DATABASE_URI = (
+    f'postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@'
+    f'{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}'
+)
+
+DATABASE_CONNECTIONS = [
+    {
+        'database_name': 'my_postgres',
+        'sqlalchemy_uri':     f'postgresql://{POSTGRES_USER}:{POSTGRES_PASSWORD}@'
+    f'{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}',
+    },
+
+    # Add more database connections as needed
+]