Fixed script, added base LLM eval option
This commit is contained in:
Vasilije
parent
e2c712fc03
commit
05cb378b83
11 changed files with 571 additions and 306 deletions
|
|
@ -44,6 +44,7 @@ RUN apt-get update -q && \
|
|||
WORKDIR /app
|
||||
COPY . /app
|
||||
COPY entrypoint.sh /app/entrypoint.sh
|
||||
COPY scripts/create_database.py /app/create_database.py
|
||||
RUN chmod +x /app/entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||
300
level_3/api.py
300
level_3/api.py
|
|
@ -7,6 +7,8 @@ from fastapi import FastAPI
|
|||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from level_3.database.database import AsyncSessionLocal
|
||||
from level_3.database.database_crud import session_scope
|
||||
from vectorstore_manager import Memory
|
||||
from dotenv import load_dotenv
|
||||
# Set up logging
|
||||
|
|
@ -62,21 +64,28 @@ def memory_factory(memory_type):
|
|||
# files: List[UploadFile] = File(...),
|
||||
):
|
||||
try:
|
||||
logging.info(" Init PDF processing")
|
||||
|
||||
logging.info(" Adding to Memory ")
|
||||
decoded_payload = payload.payload
|
||||
async with session_scope(session=AsyncSessionLocal()) as session:
|
||||
|
||||
Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
memory = await Memory.create_memory(decoded_payload["user_id"], session, namespace='SEMANTICMEMORY')
|
||||
|
||||
await Memory_.async_init()
|
||||
# Adding a memory instance
|
||||
await memory.add_memory_instance(decoded_payload["memory_object"])
|
||||
|
||||
memory_class = getattr(Memory_, f"_add_{memory_type}_memory", None)
|
||||
output = await memory_class(
|
||||
observation=decoded_payload["prompt"],
|
||||
loader_settings=decoded_payload["loader_settings"],
|
||||
params=decoded_payload["params"],
|
||||
)
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
# Managing memory attributes
|
||||
existing_user = await Memory.check_existing_user(decoded_payload["user_id"], session)
|
||||
await memory.manage_memory_attributes(existing_user)
|
||||
await memory.add_dynamic_memory_class(decoded_payload["memory_object"], decoded_payload["memory_object"].upper())
|
||||
memory_class = decoded_payload["memory_object"] + "_class"
|
||||
dynamic_memory_class = getattr(memory, memory_class.lower(), None)
|
||||
|
||||
await memory.add_method_to_class(dynamic_memory_class, 'add_memories')
|
||||
# await memory.add_method_to_class(memory.semanticmemory_class, 'fetch_memories')
|
||||
output = await memory.dynamic_method_call(dynamic_memory_class, 'add_memories',
|
||||
observation='some_observation', params=decoded_payload["params"],
|
||||
loader_settings=decoded_payload["loader_settings"])
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
|
||||
except Exception as e:
|
||||
return JSONResponse(
|
||||
|
|
@ -89,15 +98,27 @@ def memory_factory(memory_type):
|
|||
# files: List[UploadFile] = File(...),
|
||||
):
|
||||
try:
|
||||
logging.info(" Adding to Memory ")
|
||||
decoded_payload = payload.payload
|
||||
async with session_scope(session=AsyncSessionLocal()) as session:
|
||||
|
||||
Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
memory = await Memory.create_memory(decoded_payload["user_id"], session, namespace='SEMANTICMEMORY')
|
||||
|
||||
await Memory_.async_init()
|
||||
# Adding a memory instance
|
||||
await memory.add_memory_instance(decoded_payload["memory_object"])
|
||||
|
||||
memory_class = getattr(Memory_, f"_fetch_{memory_type}_memory", None)
|
||||
output = memory_class(observation=decoded_payload["prompt"])
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
# Managing memory attributes
|
||||
existing_user = await Memory.check_existing_user(decoded_payload["user_id"], session)
|
||||
await memory.manage_memory_attributes(existing_user)
|
||||
await memory.add_dynamic_memory_class(decoded_payload["memory_object"], decoded_payload["memory_object"].upper())
|
||||
memory_class = decoded_payload["memory_object"] + "_class"
|
||||
dynamic_memory_class = getattr(memory, memory_class.lower(), None)
|
||||
|
||||
await memory.add_method_to_class(dynamic_memory_class, 'add_memories')
|
||||
# await memory.add_method_to_class(memory.semanticmemory_class, 'fetch_memories')
|
||||
output = await memory.dynamic_method_call(dynamic_memory_class, 'fetch_memories',
|
||||
observation=decoded_payload['observation'])
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
|
||||
except Exception as e:
|
||||
return JSONResponse(
|
||||
|
|
@ -110,141 +131,152 @@ def memory_factory(memory_type):
|
|||
# files: List[UploadFile] = File(...),
|
||||
):
|
||||
try:
|
||||
logging.info(" Adding to Memory ")
|
||||
decoded_payload = payload.payload
|
||||
async with session_scope(session=AsyncSessionLocal()) as session:
|
||||
|
||||
Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
memory = await Memory.create_memory(decoded_payload["user_id"], session, namespace='SEMANTICMEMORY')
|
||||
|
||||
await Memory_.async_init()
|
||||
# Adding a memory instance
|
||||
await memory.add_memory_instance(decoded_payload["memory_object"])
|
||||
|
||||
memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
output = memory_class(observation=decoded_payload["prompt"])
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
# Managing memory attributes
|
||||
existing_user = await Memory.check_existing_user(decoded_payload["user_id"], session)
|
||||
await memory.manage_memory_attributes(existing_user)
|
||||
await memory.add_dynamic_memory_class(decoded_payload["memory_object"], decoded_payload["memory_object"].upper())
|
||||
memory_class = decoded_payload["memory_object"] + "_class"
|
||||
dynamic_memory_class = getattr(memory, memory_class.lower(), None)
|
||||
|
||||
await memory.add_method_to_class(dynamic_memory_class, 'delete_memories')
|
||||
# await memory.add_method_to_class(memory.semanticmemory_class, 'fetch_memories')
|
||||
output = await memory.dynamic_method_call(dynamic_memory_class, 'delete_memories',
|
||||
namespace=decoded_payload["memory_object"].upper())
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
|
||||
except Exception as e:
|
||||
return JSONResponse(
|
||||
content={"response": {"error": str(e)}}, status_code=503
|
||||
)
|
||||
|
||||
|
||||
memory_list = ["episodic", "buffer", "semantic"]
|
||||
for memory_type in memory_list:
|
||||
memory_factory(memory_type)
|
||||
|
||||
|
||||
@app.get("/available-buffer-actions", response_model=dict)
|
||||
async def available_buffer_actions(
|
||||
payload: Payload,
|
||||
# files: List[UploadFile] = File(...),
|
||||
):
|
||||
try:
|
||||
decoded_payload = payload.payload
|
||||
|
||||
Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
|
||||
await Memory_.async_init()
|
||||
|
||||
# memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
output = await Memory_._available_operations()
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
|
||||
except Exception as e:
|
||||
return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
|
||||
# @app.get("/available-buffer-actions", response_model=dict)
|
||||
# async def available_buffer_actions(
|
||||
# payload: Payload,
|
||||
# # files: List[UploadFile] = File(...),
|
||||
# ):
|
||||
# try:
|
||||
# decoded_payload = payload.payload
|
||||
#
|
||||
# Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
#
|
||||
# await Memory_.async_init()
|
||||
#
|
||||
# # memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
# output = await Memory_._available_operations()
|
||||
# return JSONResponse(content={"response": output}, status_code=200)
|
||||
#
|
||||
# except Exception as e:
|
||||
# return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
|
||||
|
||||
|
||||
@app.post("/run-buffer", response_model=dict)
|
||||
async def run_buffer(
|
||||
payload: Payload,
|
||||
# files: List[UploadFile] = File(...),
|
||||
):
|
||||
try:
|
||||
decoded_payload = payload.payload
|
||||
|
||||
Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
|
||||
await Memory_.async_init()
|
||||
|
||||
# memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
output = await Memory_._run_main_buffer(
|
||||
user_input=decoded_payload["prompt"], params=decoded_payload["params"], attention_modulators=decoded_payload["attention_modulators"]
|
||||
)
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
|
||||
except Exception as e:
|
||||
return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
|
||||
|
||||
|
||||
@app.post("/buffer/create-context", response_model=dict)
|
||||
async def create_context(
|
||||
payload: Payload,
|
||||
# files: List[UploadFile] = File(...),
|
||||
):
|
||||
try:
|
||||
decoded_payload = payload.payload
|
||||
|
||||
Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
|
||||
await Memory_.async_init()
|
||||
|
||||
# memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
output = await Memory_._create_buffer_context(
|
||||
user_input=decoded_payload["prompt"], params=decoded_payload["params"], attention_modulators=decoded_payload["attention_modulators"]
|
||||
)
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
|
||||
except Exception as e:
|
||||
return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
|
||||
|
||||
|
||||
@app.post("/buffer/get-tasks", response_model=dict)
|
||||
async def create_context(
|
||||
payload: Payload,
|
||||
# files: List[UploadFile] = File(...),
|
||||
):
|
||||
try:
|
||||
decoded_payload = payload.payload
|
||||
|
||||
Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
|
||||
await Memory_.async_init()
|
||||
|
||||
# memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
output = await Memory_._get_task_list(
|
||||
user_input=decoded_payload["prompt"], params=decoded_payload["params"], attention_modulators=decoded_payload["attention_modulators"]
|
||||
)
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
|
||||
except Exception as e:
|
||||
return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
|
||||
|
||||
|
||||
@app.post("/buffer/provide-feedback", response_model=dict)
|
||||
async def provide_feedback(
|
||||
payload: Payload,
|
||||
# files: List[UploadFile] = File(...),
|
||||
):
|
||||
try:
|
||||
decoded_payload = payload.payload
|
||||
|
||||
Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
|
||||
await Memory_.async_init()
|
||||
|
||||
# memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
if decoded_payload["total_score"] is None:
|
||||
|
||||
output = await Memory_._provide_feedback(
|
||||
user_input=decoded_payload["prompt"], params=decoded_payload["params"], attention_modulators=None, total_score=decoded_payload["total_score"]
|
||||
)
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
else:
|
||||
output = await Memory_._provide_feedback(
|
||||
user_input=decoded_payload["prompt"], params=decoded_payload["params"], attention_modulators=decoded_payload["attention_modulators"], total_score=None
|
||||
)
|
||||
return JSONResponse(content={"response": output}, status_code=200)
|
||||
|
||||
|
||||
except Exception as e:
|
||||
return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
|
||||
# @app.post("/run-buffer", response_model=dict)
|
||||
# async def run_buffer(
|
||||
# payload: Payload,
|
||||
# # files: List[UploadFile] = File(...),
|
||||
# ):
|
||||
# try:
|
||||
# decoded_payload = payload.payload
|
||||
#
|
||||
# Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
#
|
||||
# await Memory_.async_init()
|
||||
#
|
||||
# # memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
# output = await Memory_._run_main_buffer(
|
||||
# user_input=decoded_payload["prompt"], params=decoded_payload["params"], attention_modulators=decoded_payload["attention_modulators"]
|
||||
# )
|
||||
# return JSONResponse(content={"response": output}, status_code=200)
|
||||
#
|
||||
# except Exception as e:
|
||||
# return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
|
||||
#
|
||||
#
|
||||
# @app.post("/buffer/create-context", response_model=dict)
|
||||
# async def create_context(
|
||||
# payload: Payload,
|
||||
# # files: List[UploadFile] = File(...),
|
||||
# ):
|
||||
# try:
|
||||
# decoded_payload = payload.payload
|
||||
#
|
||||
# Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
#
|
||||
# await Memory_.async_init()
|
||||
#
|
||||
# # memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
# output = await Memory_._create_buffer_context(
|
||||
# user_input=decoded_payload["prompt"], params=decoded_payload["params"], attention_modulators=decoded_payload["attention_modulators"]
|
||||
# )
|
||||
# return JSONResponse(content={"response": output}, status_code=200)
|
||||
#
|
||||
# except Exception as e:
|
||||
# return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
|
||||
#
|
||||
#
|
||||
# @app.post("/buffer/get-tasks", response_model=dict)
|
||||
# async def create_context(
|
||||
# payload: Payload,
|
||||
# # files: List[UploadFile] = File(...),
|
||||
# ):
|
||||
# try:
|
||||
# decoded_payload = payload.payload
|
||||
#
|
||||
# Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
#
|
||||
# await Memory_.async_init()
|
||||
#
|
||||
# # memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
# output = await Memory_._get_task_list(
|
||||
# user_input=decoded_payload["prompt"], params=decoded_payload["params"], attention_modulators=decoded_payload["attention_modulators"]
|
||||
# )
|
||||
# return JSONResponse(content={"response": output}, status_code=200)
|
||||
#
|
||||
# except Exception as e:
|
||||
# return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
|
||||
#
|
||||
#
|
||||
# @app.post("/buffer/provide-feedback", response_model=dict)
|
||||
# async def provide_feedback(
|
||||
# payload: Payload,
|
||||
# # files: List[UploadFile] = File(...),
|
||||
# ):
|
||||
# try:
|
||||
# decoded_payload = payload.payload
|
||||
#
|
||||
# Memory_ = Memory(user_id=decoded_payload["user_id"])
|
||||
#
|
||||
# await Memory_.async_init()
|
||||
#
|
||||
# # memory_class = getattr(Memory_, f"_delete_{memory_type}_memory", None)
|
||||
# if decoded_payload["total_score"] is None:
|
||||
#
|
||||
# output = await Memory_._provide_feedback(
|
||||
# user_input=decoded_payload["prompt"], params=decoded_payload["params"], attention_modulators=None, total_score=decoded_payload["total_score"]
|
||||
# )
|
||||
# return JSONResponse(content={"response": output}, status_code=200)
|
||||
# else:
|
||||
# output = await Memory_._provide_feedback(
|
||||
# user_input=decoded_payload["prompt"], params=decoded_payload["params"], attention_modulators=decoded_payload["attention_modulators"], total_score=None
|
||||
# )
|
||||
# return JSONResponse(content={"response": output}, status_code=200)
|
||||
#
|
||||
#
|
||||
# except Exception as e:
|
||||
# return JSONResponse(content={"response": {"error": str(e)}}, status_code=503)
|
||||
def start_api_server(host: str = "0.0.0.0", port: int = 8000):
|
||||
"""
|
||||
Start the API server using uvicorn.
|
||||
|
|
|
|||
|
|
@ -23,5 +23,4 @@ async def session_scope(session):
|
|||
async def add_entity(session, entity):
|
||||
async with session_scope(session) as s: # Use your async session_scope
|
||||
s.add(entity) # No need to commit; session_scope takes care of it
|
||||
s.commit()
|
||||
return "Successfully added entity"
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
#!/bin/bash
|
||||
export ENVIRONMENT
|
||||
python fetch_secret.py
|
||||
python scripts/create_database.py
|
||||
python create_database.py
|
||||
|
||||
# Start Gunicorn
|
||||
gunicorn -w 2 -k uvicorn.workers.UvicornWorker -t 120 --bind=0.0.0.0:8000 --bind=0.0.0.0:443 --log-level debug api:app
|
||||
182
level_3/poetry.lock
generated
182
level_3/poetry.lock
generated
|
|
@ -122,6 +122,21 @@ files = [
|
|||
[package.dependencies]
|
||||
frozenlist = ">=1.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "ansi2html"
|
||||
version = "1.8.0"
|
||||
description = ""
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "ansi2html-1.8.0-py3-none-any.whl", hash = "sha256:ef9cc9682539dbe524fbf8edad9c9462a308e04bce1170c32daa8fdfd0001785"},
|
||||
{file = "ansi2html-1.8.0.tar.gz", hash = "sha256:38b82a298482a1fa2613f0f9c9beb3db72a8f832eeac58eb2e47bf32cd37f6d5"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
docs = ["Sphinx", "setuptools-scm", "sphinx-rtd-theme"]
|
||||
test = ["pytest", "pytest-cov"]
|
||||
|
||||
[[package]]
|
||||
name = "anyio"
|
||||
version = "3.7.1"
|
||||
|
|
@ -686,6 +701,73 @@ files = [
|
|||
{file = "cymem-2.0.7.tar.gz", hash = "sha256:e6034badb5dd4e10344211c81f16505a55553a7164adc314c75bd80cf07e57a8"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dash"
|
||||
version = "2.14.0"
|
||||
description = "A Python framework for building reactive web-apps. Developed by Plotly."
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "dash-2.14.0-py3-none-any.whl", hash = "sha256:b88ffb53cb1bf54c12780ecf89943bb901c8914b3c075002e46519c9b17d7a72"},
|
||||
{file = "dash-2.14.0.tar.gz", hash = "sha256:bd28be70be24ae1d1f764b8217a03da35e9ed895406686d24dfb6ed4e331e5a9"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
ansi2html = "*"
|
||||
dash-core-components = "2.0.0"
|
||||
dash-html-components = "2.0.0"
|
||||
dash-table = "5.0.0"
|
||||
Flask = ">=1.0.4,<2.3.0"
|
||||
importlib-metadata = {version = "*", markers = "python_version >= \"3.7\""}
|
||||
nest-asyncio = "*"
|
||||
plotly = ">=5.0.0"
|
||||
requests = "*"
|
||||
retrying = "*"
|
||||
setuptools = "*"
|
||||
typing-extensions = ">=4.1.1"
|
||||
Werkzeug = "<2.3.0"
|
||||
|
||||
[package.extras]
|
||||
celery = ["celery[redis] (>=5.1.2)", "importlib-metadata (<5)", "redis (>=3.5.3)"]
|
||||
ci = ["black (==21.6b0)", "black (==22.3.0)", "dash-dangerously-set-inner-html", "dash-flow-example (==0.0.5)", "flake8 (==3.9.2)", "flaky (==3.7.0)", "flask-talisman (==1.0.0)", "isort (==4.3.21)", "jupyterlab (<4.0.0)", "mimesis", "mock (==4.0.3)", "numpy (<=1.25.2)", "openpyxl", "orjson (==3.5.4)", "orjson (==3.6.7)", "pandas (==1.1.5)", "pandas (>=1.4.0)", "preconditions", "pyarrow", "pyarrow (<3)", "pylint (==2.13.5)", "pytest-mock", "pytest-rerunfailures", "pytest-sugar (==0.9.6)", "xlrd (<2)", "xlrd (>=2.0.1)"]
|
||||
compress = ["flask-compress"]
|
||||
dev = ["PyYAML (>=5.4.1)", "coloredlogs (>=15.0.1)", "fire (>=0.4.0)"]
|
||||
diskcache = ["diskcache (>=5.2.1)", "multiprocess (>=0.70.12)", "psutil (>=5.8.0)"]
|
||||
testing = ["beautifulsoup4 (>=4.8.2)", "cryptography (<3.4)", "dash-testing-stub (>=0.0.2)", "lxml (>=4.6.2)", "multiprocess (>=0.70.12)", "percy (>=2.0.2)", "psutil (>=5.8.0)", "pytest (>=6.0.2)", "requests[security] (>=2.21.0)", "selenium (>=3.141.0,<=4.2.0)", "waitress (>=1.4.4)"]
|
||||
|
||||
[[package]]
|
||||
name = "dash-core-components"
|
||||
version = "2.0.0"
|
||||
description = "Core component suite for Dash"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "dash_core_components-2.0.0-py3-none-any.whl", hash = "sha256:52b8e8cce13b18d0802ee3acbc5e888cb1248a04968f962d63d070400af2e346"},
|
||||
{file = "dash_core_components-2.0.0.tar.gz", hash = "sha256:c6733874af975e552f95a1398a16c2ee7df14ce43fa60bb3718a3c6e0b63ffee"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dash-html-components"
|
||||
version = "2.0.0"
|
||||
description = "Vanilla HTML components for Dash"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "dash_html_components-2.0.0-py3-none-any.whl", hash = "sha256:b42cc903713c9706af03b3f2548bda4be7307a7cf89b7d6eae3da872717d1b63"},
|
||||
{file = "dash_html_components-2.0.0.tar.gz", hash = "sha256:8703a601080f02619a6390998e0b3da4a5daabe97a1fd7a9cebc09d015f26e50"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dash-table"
|
||||
version = "5.0.0"
|
||||
description = "Dash table"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "dash_table-5.0.0-py3-none-any.whl", hash = "sha256:19036fa352bb1c11baf38068ec62d172f0515f73ca3276c79dee49b95ddc16c9"},
|
||||
{file = "dash_table-5.0.0.tar.gz", hash = "sha256:18624d693d4c8ef2ddec99a6f167593437a7ea0bf153aa20f318c170c5bc7308"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dataclasses-json"
|
||||
version = "0.5.9"
|
||||
|
|
@ -1005,6 +1087,27 @@ typing-extensions = {version = ">=4.7.1", markers = "python_version < \"3.11\""}
|
|||
docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"]
|
||||
testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"]
|
||||
|
||||
[[package]]
|
||||
name = "flask"
|
||||
version = "2.2.5"
|
||||
description = "A simple framework for building complex web applications."
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "Flask-2.2.5-py3-none-any.whl", hash = "sha256:58107ed83443e86067e41eff4631b058178191a355886f8e479e347fa1285fdf"},
|
||||
{file = "Flask-2.2.5.tar.gz", hash = "sha256:edee9b0a7ff26621bd5a8c10ff484ae28737a2410d99b0bb9a6850c7fb977aa0"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
click = ">=8.0"
|
||||
itsdangerous = ">=2.0"
|
||||
Jinja2 = ">=3.0"
|
||||
Werkzeug = ">=2.2.2"
|
||||
|
||||
[package.extras]
|
||||
async = ["asgiref (>=3.2)"]
|
||||
dotenv = ["python-dotenv"]
|
||||
|
||||
[[package]]
|
||||
name = "frozenlist"
|
||||
version = "1.4.0"
|
||||
|
|
@ -1424,6 +1527,25 @@ files = [
|
|||
{file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "importlib-metadata"
|
||||
version = "6.8.0"
|
||||
description = "Read metadata from Python packages"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "importlib_metadata-6.8.0-py3-none-any.whl", hash = "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb"},
|
||||
{file = "importlib_metadata-6.8.0.tar.gz", hash = "sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
zipp = ">=0.5"
|
||||
|
||||
[package.extras]
|
||||
docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"]
|
||||
perf = ["ipython"]
|
||||
testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"]
|
||||
|
||||
[[package]]
|
||||
name = "iniconfig"
|
||||
version = "2.0.0"
|
||||
|
|
@ -1435,6 +1557,17 @@ files = [
|
|||
{file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itsdangerous"
|
||||
version = "2.1.2"
|
||||
description = "Safely pass data to untrusted environments and back."
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"},
|
||||
{file = "itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jinja2"
|
||||
version = "3.1.2"
|
||||
|
|
@ -2515,6 +2648,21 @@ files = [
|
|||
greenlet = "2.0.2"
|
||||
pyee = "9.0.4"
|
||||
|
||||
[[package]]
|
||||
name = "plotly"
|
||||
version = "5.17.0"
|
||||
description = "An open-source, interactive data visualization library for Python"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "plotly-5.17.0-py2.py3-none-any.whl", hash = "sha256:7c84cdf11da162423da957bb093287134f2d6f170eb9a74f1459f825892247c3"},
|
||||
{file = "plotly-5.17.0.tar.gz", hash = "sha256:290d796bf7bab87aad184fe24b86096234c4c95dcca6ecbca02d02bdf17d3d97"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
packaging = "*"
|
||||
tenacity = ">=6.2.0"
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.3.0"
|
||||
|
|
@ -4661,6 +4809,23 @@ validators = ">=0.18.2,<=0.21.0"
|
|||
[package.extras]
|
||||
grpc = ["grpcio", "grpcio-tools"]
|
||||
|
||||
[[package]]
|
||||
name = "werkzeug"
|
||||
version = "2.2.3"
|
||||
description = "The comprehensive WSGI web application library."
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
files = [
|
||||
{file = "Werkzeug-2.2.3-py3-none-any.whl", hash = "sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612"},
|
||||
{file = "Werkzeug-2.2.3.tar.gz", hash = "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
MarkupSafe = ">=2.1.1"
|
||||
|
||||
[package.extras]
|
||||
watchdog = ["watchdog"]
|
||||
|
||||
[[package]]
|
||||
name = "wheel"
|
||||
version = "0.41.2"
|
||||
|
|
@ -4874,7 +5039,22 @@ files = [
|
|||
idna = ">=2.0"
|
||||
multidict = ">=4.0"
|
||||
|
||||
[[package]]
|
||||
name = "zipp"
|
||||
version = "3.17.0"
|
||||
description = "Backport of pathlib-compatible object wrapper for zip files"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "zipp-3.17.0-py3-none-any.whl", hash = "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31"},
|
||||
{file = "zipp-3.17.0.tar.gz", hash = "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"]
|
||||
testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"]
|
||||
|
||||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = "^3.10"
|
||||
content-hash = "b3f795baef70806c8dab1058b644d95f3c36b1c2fae53a7e5784a633731bc268"
|
||||
content-hash = "d022983c3e6224970ff0e1d90bffb2e303c6a474ff1fc87345f0227eae2c7b29"
|
||||
|
|
|
|||
|
|
@ -40,13 +40,14 @@ weaviate-client = "^3.22.1"
|
|||
python-multipart = "^0.0.6"
|
||||
deep-translator = "^1.11.4"
|
||||
humanize = "^4.8.0"
|
||||
deepeval = "^0.20.6"
|
||||
deepeval = "^0.20.4"
|
||||
pymupdf = "^1.23.3"
|
||||
psycopg2 = "^2.9.8"
|
||||
llama-index = "^0.8.39.post2"
|
||||
llama-hub = "^0.0.34"
|
||||
sqlalchemy = "^2.0.21"
|
||||
asyncpg = "^0.28.0"
|
||||
dash = "^2.14.0"
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,38 +1,56 @@
|
|||
import argparse
|
||||
import json
|
||||
from enum import Enum
|
||||
import sys
|
||||
import os
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
from deepeval.metrics.overall_score import OverallScoreMetric
|
||||
from deepeval.test_case import LLMTestCase
|
||||
from deepeval.run_test import assert_test, run_test
|
||||
from gptcache.embedding import openai
|
||||
from marvin import ai_classifier
|
||||
|
||||
import random
|
||||
import itertools
|
||||
|
||||
|
||||
import logging
|
||||
import string
|
||||
from enum import Enum
|
||||
|
||||
|
||||
import openai
|
||||
from deepeval.metrics.overall_score import OverallScoreMetric
|
||||
from deepeval.run_test import run_test
|
||||
from deepeval.test_case import LLMTestCase
|
||||
from marvin import ai_classifier
|
||||
from sqlalchemy.future import select
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
import marvin
|
||||
from dotenv import load_dotenv
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from database.database import engine # Ensure you have database engine defined somewhere
|
||||
from models.user import User
|
||||
from models.memory import MemoryModel
|
||||
from models.sessions import Session
|
||||
from models.testset import TestSet
|
||||
from models.testoutput import TestOutput
|
||||
from models.metadatas import MetaDatas
|
||||
from models.operation import Operation
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from database.database import engine
|
||||
from vectorstore_manager import Memory
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from database.database import AsyncSessionLocal
|
||||
from database.database_crud import session_scope
|
||||
load_dotenv()
|
||||
import ast
|
||||
import tracemalloc
|
||||
from database.database_crud import session_scope, add_entity
|
||||
|
||||
import random
|
||||
import string
|
||||
import itertools
|
||||
import logging
|
||||
import dotenv
|
||||
dotenv.load_dotenv()
|
||||
logger = logging.getLogger(__name__)
|
||||
import openai
|
||||
tracemalloc.start()
|
||||
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
import uuid
|
||||
|
||||
load_dotenv()
|
||||
|
||||
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
|
||||
marvin.settings.openai.api_key = os.environ.get("OPENAI_API_KEY")
|
||||
from vectordb.basevectordb import BaseMemory
|
||||
from vectorstore_manager import Memory
|
||||
import asyncio
|
||||
|
||||
from database.database_crud import session_scope
|
||||
from database.database import AsyncSessionLocal
|
||||
openai.api_key = os.getenv("OPENAI_API_KEY", "")
|
||||
|
||||
|
||||
async def retrieve_latest_test_case(session, user_id, memory_id):
|
||||
try:
|
||||
# Use await with session.execute() and row.fetchone() or row.all() for async query execution
|
||||
|
|
@ -43,13 +61,14 @@ async def retrieve_latest_test_case(session, user_id, memory_id):
|
|||
)
|
||||
return result.scalar_one_or_none() # scalar_one_or_none() is a non-blocking call
|
||||
except Exception as e:
|
||||
logger.error(f"An error occurred while retrieving the latest test case: {str(e)}")
|
||||
logging.error(f"An error occurred while retrieving the latest test case: {str(e)}")
|
||||
return None
|
||||
|
||||
async def add_entity(session, entity):
|
||||
async with session_scope(session) as s: # Use your async session_scope
|
||||
s.add(entity) # No need to commit; session_scope takes care of it
|
||||
s.commit()
|
||||
|
||||
return "Successfully added entity"
|
||||
|
||||
async def retrieve_job_by_id(session, user_id, job_id):
|
||||
|
|
@ -61,7 +80,7 @@ async def retrieve_job_by_id(session, user_id, job_id):
|
|||
)
|
||||
return result.scalar_one_or_none()
|
||||
except Exception as e:
|
||||
logger.error(f"An error occurred while retrieving the job: {str(e)}")
|
||||
logging.error(f"An error occurred while retrieving the job: {str(e)}")
|
||||
return None
|
||||
|
||||
async def fetch_job_id(session, user_id=None, memory_id=None, job_id=None):
|
||||
|
|
@ -73,7 +92,7 @@ async def fetch_job_id(session, user_id=None, memory_id=None, job_id=None):
|
|||
)
|
||||
return result.scalar_one_or_none()
|
||||
except Exception as e:
|
||||
logger.error(f"An error occurred: {str(e)}")
|
||||
logging.error(f"An error occurred: {str(e)}")
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -87,7 +106,7 @@ async def fetch_test_set_id(session, user_id, id):
|
|||
)
|
||||
return result.scalar_one_or_none() # scalar_one_or_none() is a non-blocking call
|
||||
except Exception as e:
|
||||
logger.error(f"An error occurred while retrieving the test set: {str(e)}")
|
||||
logging.error(f"An error occurred while retrieving the test set: {str(e)}")
|
||||
return None
|
||||
|
||||
# Adding "embeddings" to the parameter variants function
|
||||
|
|
@ -178,20 +197,20 @@ async def generate_chatgpt_output(query:str, context:str=None):
|
|||
# print(llm_output)
|
||||
return llm_output
|
||||
|
||||
async def eval_test(query=None, output=None, expected_output=None, context=None):
|
||||
# query = "How does photosynthesis work?"
|
||||
# output = "Photosynthesis is the process by which green plants and some other organisms use sunlight to synthesize foods with the help of chlorophyll pigment."
|
||||
# expected_output = "Photosynthesis is the process by which green plants and some other organisms use sunlight to synthesize food with the help of chlorophyll pigment."
|
||||
# context = "Biology"
|
||||
async def eval_test(query=None, output=None, expected_output=None, context=None, synthetic_test_set=False):
|
||||
|
||||
result_output = await generate_chatgpt_output(query, context)
|
||||
|
||||
test_case = LLMTestCase(
|
||||
query=query,
|
||||
output=result_output,
|
||||
expected_output=expected_output,
|
||||
context=context,
|
||||
if synthetic_test_set:
|
||||
test_case = synthetic_test_set
|
||||
else:
|
||||
|
||||
)
|
||||
test_case = LLMTestCase(
|
||||
query=query,
|
||||
output=result_output,
|
||||
expected_output=expected_output,
|
||||
context=context,
|
||||
)
|
||||
metric = OverallScoreMetric()
|
||||
|
||||
# If you want to run the test
|
||||
|
|
@ -230,12 +249,11 @@ def data_location_route(data_string: str):
|
|||
return LocationRoute(data_string).name
|
||||
|
||||
|
||||
def dynamic_test_manager(data, test_set=None, user=None, params=None):
|
||||
def dynamic_test_manager(context=None):
|
||||
from deepeval.dataset import create_evaluation_query_answer_pairs
|
||||
# fetch random chunks from the document
|
||||
#feed them to the evaluation pipeline
|
||||
dataset = create_evaluation_query_answer_pairs(
|
||||
"Python is a great language for mathematical expression and machine learning.")
|
||||
dataset = create_evaluation_query_answer_pairs(openai_api_key=os.environ.get("OPENAI_API_KEY"), context= context ,n=10)
|
||||
|
||||
return dataset
|
||||
|
||||
|
|
@ -246,11 +264,12 @@ def generate_letter_uuid(length=8):
|
|||
letters = string.ascii_uppercase # A-Z
|
||||
return ''.join(random.choice(letters) for _ in range(length))
|
||||
|
||||
async def start_test(data, test_set=None, user_id=None, params=None, job_id=None, metadata=None):
|
||||
async def start_test(data, test_set=None, user_id=None, params=None, job_id=None, metadata=None, generate_test_set=False, only_llm_context=False):
|
||||
|
||||
|
||||
async with session_scope(session=AsyncSessionLocal()) as session:
|
||||
memory = await Memory.create_memory(user_id, session, namespace="SEMANTICMEMORY")
|
||||
|
||||
|
||||
job_id = await fetch_job_id(session, user_id=user_id, job_id=job_id)
|
||||
test_set_id = await fetch_test_set_id(session, user_id=user_id, id=job_id)
|
||||
|
||||
|
|
@ -266,104 +285,113 @@ async def start_test(data, test_set=None, user_id=None, params=None, job_id=None
|
|||
data_format = data_format_route(data) # Assume data_format_route is predefined
|
||||
data_location = data_location_route(data) # Assume data_location_route is predefined
|
||||
test_params = generate_param_variants(
|
||||
included_params=['chunk_size', 'chunk_overlap', 'search_type'])
|
||||
included_params=['chunk_size'])
|
||||
|
||||
print("Here are the test params", str(test_params))
|
||||
|
||||
loader_settings = {
|
||||
"format": f"{data_format}",
|
||||
"source": f"{data_location}",
|
||||
"path": "https://www.ibiblio.org/ebooks/London/Call%20of%20Wild.pdf"
|
||||
"path": data
|
||||
}
|
||||
|
||||
async def run_test(test, loader_settings, metadata):
|
||||
test_id = str(generate_letter_uuid()) + "_" + "SEMANTICEMEMORY"
|
||||
|
||||
async def run_test(test, loader_settings, metadata, test_id=None,only_llm_context=False):
|
||||
|
||||
if test_id is None:
|
||||
test_id = str(generate_letter_uuid()) + "_" +"SEMANTICMEMORY"
|
||||
memory = await Memory.create_memory(user_id, session, namespace="SEMANTICMEMORY")
|
||||
await memory.add_memory_instance("ExampleMemory")
|
||||
existing_user = await Memory.check_existing_user(user_id, session)
|
||||
await memory.manage_memory_attributes(existing_user)
|
||||
test_class = test_id + "_class"
|
||||
await memory.add_dynamic_memory_class(test_id.lower(), test_id)
|
||||
dynamic_memory_class = getattr(memory, test_class.lower(), None)
|
||||
|
||||
# Assuming add_method_to_class and dynamic_method_call are predefined methods in Memory class
|
||||
methods_to_add = ['add_memories', 'fetch_memories', 'delete_memories']
|
||||
|
||||
if dynamic_memory_class is not None:
|
||||
await memory.add_method_to_class(dynamic_memory_class, 'add_memories')
|
||||
else:
|
||||
print(f"No attribute named {test_class.lower()} in memory.")
|
||||
|
||||
if dynamic_memory_class is not None:
|
||||
await memory.add_method_to_class(dynamic_memory_class, 'fetch_memories')
|
||||
for method_name in methods_to_add:
|
||||
await memory.add_method_to_class(dynamic_memory_class, method_name)
|
||||
print(f"Memory method {method_name} has been added")
|
||||
else:
|
||||
print(f"No attribute named {test_class.lower()} in memory.")
|
||||
|
||||
print(f"Trying to access: {test_class.lower()}")
|
||||
print("Available memory classes:", await memory.list_memory_classes())
|
||||
|
||||
print(f"Trying to check: ", test)
|
||||
loader_settings.update(test)
|
||||
|
||||
|
||||
async def run_load_test_element(test, loader_settings, metadata, test_id):
|
||||
|
||||
test_class = test_id + "_class"
|
||||
# memory.test_class
|
||||
|
||||
await memory.add_dynamic_memory_class(test_id.lower(), test_id)
|
||||
dynamic_memory_class = getattr(memory, test_class.lower(), None)
|
||||
if dynamic_memory_class is not None:
|
||||
await memory.add_method_to_class(dynamic_memory_class, 'add_memories')
|
||||
else:
|
||||
print(f"No attribute named {test_class.lower()} in memory.")
|
||||
|
||||
if dynamic_memory_class is not None:
|
||||
await memory.add_method_to_class(dynamic_memory_class, 'fetch_memories')
|
||||
else:
|
||||
print(f"No attribute named {test_class.lower()} in memory.")
|
||||
if test:
|
||||
loader_settings.update(test)
|
||||
test_class = test_id + "_class"
|
||||
dynamic_memory_class = getattr(memory, test_class.lower(), None)
|
||||
async def run_load_test_element( loader_settings=loader_settings, metadata=metadata, test_id=test_id, test_set=test_set):
|
||||
|
||||
print(f"Trying to access: {test_class.lower()}")
|
||||
print("Available memory classes:", await memory.list_memory_classes())
|
||||
|
||||
print(f"Trying to check: ", test)
|
||||
# print("Here is the loader settings", str(loader_settings))
|
||||
# print("Here is the medatadata", str(metadata))
|
||||
load_action = await memory.dynamic_method_call(dynamic_memory_class, 'add_memories',
|
||||
observation='some_observation', params=metadata,
|
||||
await memory.dynamic_method_call(dynamic_memory_class, 'add_memories',
|
||||
observation='Observation loaded', params=metadata,
|
||||
loader_settings=loader_settings)
|
||||
async def run_search_eval_element(test_item, test_id):
|
||||
return "Loaded test element"
|
||||
async def run_search_element(test_item, test_id):
|
||||
retrieve_action = await memory.dynamic_method_call(dynamic_memory_class, 'fetch_memories',
|
||||
observation=str(test_item["question"]))
|
||||
print("Here is the test result", str(retrieve_action["data"]['Get'][test_id][0]["text"]))
|
||||
return retrieve_action["data"]['Get'][test_id][0]["text"]
|
||||
|
||||
async def run_eval(test_item, search_result):
|
||||
test_eval= await eval_test(query=test_item["question"], expected_output=test_item["answer"],
|
||||
context=str(search_result))
|
||||
return test_eval
|
||||
async def run_generate_test_set( test_id):
|
||||
|
||||
test_class = test_id + "_class"
|
||||
await memory.add_dynamic_memory_class(test_id.lower(), test_id)
|
||||
# await memory.add_dynamic_memory_class(test_id.lower(), test_id)
|
||||
dynamic_memory_class = getattr(memory, test_class.lower(), None)
|
||||
|
||||
print(dynamic_memory_class)
|
||||
retrieve_action = await memory.dynamic_method_call(dynamic_memory_class, 'fetch_memories',
|
||||
observation=test_item["question"],
|
||||
search_type=test_item["search_type"])
|
||||
test_result = await eval_test(query=test_item["question"], expected_output=test_item["answer"],
|
||||
context=str(retrieve_action))
|
||||
print(test_result)
|
||||
delete_mems = await memory.dynamic_method_call(dynamic_memory_class, 'delete_memories',
|
||||
namespace=test_id)
|
||||
return test_result
|
||||
test_load_pipeline = await asyncio.gather(
|
||||
*(run_load_test_element(test_item,loader_settings, metadata, test_id) for test_item in test_set)
|
||||
)
|
||||
observation="Generate a short summary of this document",
|
||||
search_type="generative")
|
||||
return dynamic_test_manager(retrieve_action)
|
||||
test_eval_pipeline =[]
|
||||
|
||||
test_eval_pipeline = await asyncio.gather(
|
||||
*(run_search_eval_element(test_item, test_id) for test_item in test_set)
|
||||
)
|
||||
logging.info("Results of the eval pipeline %s", str(test_eval_pipeline))
|
||||
await add_entity(session, TestOutput(id=test_id, user_id=user_id, test_results=str(test_eval_pipeline)))
|
||||
|
||||
# # Gather and run all tests in parallel
|
||||
results = await asyncio.gather(
|
||||
*(run_test(test, loader_settings, metadata) for test in test_params)
|
||||
)
|
||||
if only_llm_context:
|
||||
for test_qa in test_set:
|
||||
context=""
|
||||
test_result = await run_eval(test_qa, context)
|
||||
test_eval_pipeline.append(test_result)
|
||||
if generate_test_set is True:
|
||||
synthetic_test_set = run_generate_test_set(test_id)
|
||||
else:
|
||||
pass
|
||||
if test_set:
|
||||
logging.info("Loading and evaluating test set")
|
||||
await run_load_test_element(loader_settings, metadata, test_id, test_set)
|
||||
for test_qa in test_set:
|
||||
result = await run_search_element(test_qa, test_id)
|
||||
test_result = await run_eval(test_qa, result)
|
||||
test_eval_pipeline.append( test_result)
|
||||
|
||||
else:
|
||||
pass
|
||||
|
||||
await memory.dynamic_method_call(dynamic_memory_class, 'delete_memories',
|
||||
namespace=test_id)
|
||||
|
||||
return test_eval_pipeline
|
||||
|
||||
results = []
|
||||
if only_llm_context:
|
||||
result = await run_test(test= None, loader_settings=loader_settings, metadata=metadata, only_llm_context=only_llm_context)
|
||||
results.append(result)
|
||||
for param in test_params:
|
||||
result = await run_test(param, loader_settings, metadata,only_llm_context=only_llm_context)
|
||||
results.append(result)
|
||||
|
||||
print(results)
|
||||
|
||||
return results
|
||||
|
||||
async def main():
|
||||
|
||||
params = {
|
||||
metadata = {
|
||||
"version": "1.0",
|
||||
"agreement_id": "AG123456",
|
||||
"privacy_policy": "https://example.com/privacy",
|
||||
|
|
@ -399,7 +427,9 @@ async def main():
|
|||
"answer": "Buck becomes the leader after defeating the original leader, Spitz, in a fight."
|
||||
}
|
||||
]
|
||||
result = await start_test("https://www.ibiblio.org/ebooks/London/Call%20of%20Wild.pdf", test_set=test_set, user_id="666", params=None, metadata=params)
|
||||
# "https://www.ibiblio.org/ebooks/London/Call%20of%20Wild.pdf"
|
||||
#http://public-library.uk/ebooks/59/83.pdf
|
||||
result = await start_test("http://public-library.uk/ebooks/59/83.pdf", test_set=test_set, user_id="676", params=None, metadata=metadata)
|
||||
#
|
||||
# parser = argparse.ArgumentParser(description="Run tests against a document.")
|
||||
# parser.add_argument("--url", required=True, help="URL of the document to test.")
|
||||
|
|
@ -441,10 +471,24 @@ async def main():
|
|||
# #clean up params here
|
||||
# await start_test(args.url, test_set, args.user_id, params=None, metadata=metadata)
|
||||
if __name__ == "__main__":
|
||||
import asyncio
|
||||
|
||||
asyncio.run(main())
|
||||
|
||||
# delete_mems = await memory.dynamic_method_call(dynamic_memory_class, 'delete_memories',
|
||||
# namespace=test_id)
|
||||
# test_load_pipeline = await asyncio.gather(
|
||||
# *(run_load_test_element(test_item,loader_settings, metadata, test_id) for test_item in test_set)
|
||||
# )
|
||||
#
|
||||
# test_eval_pipeline = await asyncio.gather(
|
||||
# *(run_search_eval_element(test_item, test_id) for test_item in test_set)
|
||||
# )
|
||||
# logging.info("Results of the eval pipeline %s", str(test_eval_pipeline))
|
||||
# await add_entity(session, TestOutput(id=test_id, user_id=user_id, test_results=str(test_eval_pipeline)))
|
||||
# return test_eval_pipeline
|
||||
|
||||
|
||||
|
||||
# # Gather and run all tests in parallel
|
||||
# results = await asyncio.gather(
|
||||
# *(run_testo(test, loader_settings, metadata) for test in test_params)
|
||||
# )
|
||||
# return results
|
||||
|
|
@ -202,6 +202,7 @@ class BaseMemory:
|
|||
return field_mapping[field_type](**kwargs)
|
||||
|
||||
# Dynamic Schema Creation
|
||||
params['user_id'] = self.user_id
|
||||
|
||||
|
||||
schema_instance = self.create_dynamic_schema(params) # Always creating Str field, adjust as needed
|
||||
|
|
@ -227,6 +228,10 @@ class BaseMemory:
|
|||
namespace: Optional[str] = None,
|
||||
n_of_observations: Optional[int] = 2,
|
||||
):
|
||||
logging.info(namespace)
|
||||
logging.info(search_type)
|
||||
logging.info(params)
|
||||
logging.info(observation)
|
||||
|
||||
return await self.vector_db.fetch_memories(
|
||||
observation=observation, search_type= search_type, params=params,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from vectordb.chunkers.chunkers import chunk_data
|
|||
from llama_hub.file.base import SimpleDirectoryReader
|
||||
|
||||
import requests
|
||||
def _document_loader( observation: str, loader_settings: dict):
|
||||
async def _document_loader( observation: str, loader_settings: dict):
|
||||
# Check the format of the document
|
||||
document_format = loader_settings.get("format", "text")
|
||||
loader_strategy = loader_settings.get("strategy", "VANILLA")
|
||||
|
|
|
|||
|
|
@ -121,17 +121,18 @@ class WeaviateVectorDB(VectorDB):
|
|||
retriever = self.init_weaviate(embeddings=embeddings,namespace = namespace)
|
||||
if loader_settings:
|
||||
# Assuming _document_loader returns a list of documents
|
||||
documents = _document_loader(observation, loader_settings)
|
||||
documents = await _document_loader(observation, loader_settings)
|
||||
logging.info("here are the docs %s", str(documents))
|
||||
for doc in documents:
|
||||
document_to_load = self._stuct(doc.page_content, params, metadata_schema_class)
|
||||
print("here is the doc to load1", document_to_load)
|
||||
|
||||
logging.info("Loading document with provided loader settings %s", str(document_to_load))
|
||||
retriever.add_documents([
|
||||
Document(metadata=document_to_load[0]['metadata'], page_content=document_to_load[0]['page_content'])])
|
||||
else:
|
||||
document_to_load = self._stuct(observation, params, metadata_schema_class)
|
||||
|
||||
print("here is the doc to load2", document_to_load)
|
||||
logging.info("Loading document with defautl loader settings %s", str(document_to_load))
|
||||
retriever.add_documents([
|
||||
Document(metadata=document_to_load[0]['metadata'], page_content=document_to_load[0]['page_content'])])
|
||||
|
||||
|
|
@ -159,6 +160,8 @@ class WeaviateVectorDB(VectorDB):
|
|||
if not namespace:
|
||||
namespace = self.namespace
|
||||
|
||||
logging.info("Query on namespace %s", namespace)
|
||||
|
||||
params_user_id = {
|
||||
"path": ["user_id"],
|
||||
"operator": "Like",
|
||||
|
|
@ -181,52 +184,52 @@ class WeaviateVectorDB(VectorDB):
|
|||
|
||||
n_of_observations = kwargs.get('n_of_observations', 2)
|
||||
|
||||
try:
|
||||
if search_type == 'text':
|
||||
query_output = (
|
||||
base_query
|
||||
.with_near_text({"concepts": [observation]})
|
||||
.with_autocut(n_of_observations)
|
||||
.do()
|
||||
)
|
||||
elif search_type == 'hybrid':
|
||||
query_output = (
|
||||
base_query
|
||||
.with_hybrid(query=observation, fusion_type=HybridFusion.RELATIVE_SCORE)
|
||||
.with_autocut(n_of_observations)
|
||||
.do()
|
||||
)
|
||||
elif search_type == 'bm25':
|
||||
query_output = (
|
||||
base_query
|
||||
.with_bm25(query=observation)
|
||||
.with_autocut(n_of_observations)
|
||||
.do()
|
||||
)
|
||||
elif search_type == 'generate':
|
||||
generate_prompt = kwargs.get('generate_prompt', "")
|
||||
query_output = (
|
||||
base_query
|
||||
.with_generate(single_prompt=generate_prompt)
|
||||
.with_near_text({"concepts": [observation]})
|
||||
.with_autocut(n_of_observations)
|
||||
.do()
|
||||
)
|
||||
elif search_type == 'generate_grouped':
|
||||
generate_prompt = kwargs.get('generate_prompt', "")
|
||||
query_output = (
|
||||
base_query
|
||||
.with_generate(grouped_task=generate_prompt)
|
||||
.with_near_text({"concepts": [observation]})
|
||||
.with_autocut(n_of_observations)
|
||||
.do()
|
||||
)
|
||||
else:
|
||||
logging.error(f"Invalid search_type: {search_type}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logging.error(f"Error executing query: {str(e)}")
|
||||
# try:
|
||||
if search_type == 'text':
|
||||
query_output = (
|
||||
base_query
|
||||
.with_near_text({"concepts": [observation]})
|
||||
.with_autocut(n_of_observations)
|
||||
.do()
|
||||
)
|
||||
elif search_type == 'hybrid':
|
||||
query_output = (
|
||||
base_query
|
||||
.with_hybrid(query=observation, fusion_type=HybridFusion.RELATIVE_SCORE)
|
||||
.with_autocut(n_of_observations)
|
||||
.do()
|
||||
)
|
||||
elif search_type == 'bm25':
|
||||
query_output = (
|
||||
base_query
|
||||
.with_bm25(query=observation)
|
||||
.with_autocut(n_of_observations)
|
||||
.do()
|
||||
)
|
||||
elif search_type == 'generate':
|
||||
generate_prompt = kwargs.get('generate_prompt', "")
|
||||
query_output = (
|
||||
base_query
|
||||
.with_generate(single_prompt=observation)
|
||||
.with_near_text({"concepts": [observation]})
|
||||
.with_autocut(n_of_observations)
|
||||
.do()
|
||||
)
|
||||
elif search_type == 'generate_grouped':
|
||||
generate_prompt = kwargs.get('generate_prompt', "")
|
||||
query_output = (
|
||||
base_query
|
||||
.with_generate(grouped_task=observation)
|
||||
.with_near_text({"concepts": [observation]})
|
||||
.with_autocut(n_of_observations)
|
||||
.do()
|
||||
)
|
||||
else:
|
||||
logging.error(f"Invalid search_type: {search_type}")
|
||||
return []
|
||||
# except Exception as e:
|
||||
# logging.error(f"Error executing query: {str(e)}")
|
||||
# return []
|
||||
|
||||
return query_output
|
||||
|
||||
|
|
|
|||
|
|
@ -342,11 +342,11 @@ async def main():
|
|||
await memory.manage_memory_attributes(existing_user)
|
||||
# aeehuvyq_semanticememory_class
|
||||
|
||||
await memory.add_dynamic_memory_class('SemanticMemory', 'SEMANTICMEMORY')
|
||||
await memory.add_dynamic_memory_class('semanticmemory', 'SEMANTICMEMORY')
|
||||
await memory.add_method_to_class(memory.semanticmemory_class, 'add_memories')
|
||||
await memory.add_method_to_class(memory.semanticmemory_class, 'fetch_memories')
|
||||
sss = await memory.dynamic_method_call(memory.semanticmemory_class, 'add_memories',
|
||||
observation='some_observation', params=params, loader_settings=loader_settings)
|
||||
# sss = await memory.dynamic_method_call(memory.semanticmemory_class, 'add_memories',
|
||||
# observation='some_observation', params=params, loader_settings=loader_settings)
|
||||
|
||||
susu = await memory.dynamic_method_call(memory.semanticmemory_class, 'fetch_memories',
|
||||
observation='some_observation')
|
||||
|
|
@ -362,8 +362,8 @@ async def main():
|
|||
|
||||
|
||||
# print(sss)
|
||||
load_jack_london = await memory._add_semantic_memory(observation = "bla", loader_settings=loader_settings, params=params)
|
||||
print(load_jack_london)
|
||||
# load_jack_london = await memory._add_semantic_memory(observation = "bla", loader_settings=loader_settings, params=params)
|
||||
# print(load_jack_london)
|
||||
|
||||
modulator = {"relevance": 0.1, "frequency": 0.1}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue