gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update docs, fix issue with params, add tracking

Browse source
This commit is contained in:
Vasilije 2023-10-30 21:21:26 +01:00
parent 552a8e65f9
commit 3409d5bf61
5 changed files with 80 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
README.md
View file

@ -149,7 +149,7 @@ Send the request to the API:
```
curl -X POST -H "Content-Type: application/json" -d '{
"payload": {
"user_id": "681",
"user_id": "97980cfea0067",
"data": [".data/3ZCCCW.pdf"],
"test_set": "sample",
"params": ["chunk_size"],
@ -217,7 +217,7 @@ After that, you can run the RAG test manager from your command line.
python rag_test_manager.py \
--file ".data" \
--test_set "example_data/test_set.json" \
--user_id "666" \
--user_id "97980cfea0067" \
--params "chunk_size" "search_type" \
--metadata "example_data/metadata.json" \
--retriever_type "single_document_context"
@ -226,3 +226,6 @@ After that, you can run the RAG test manager from your command line.
Examples of metadata structure and test set are in the folder "example_data"

3
level_3/.env.template
View file

@ -6,4 +6,5 @@ POSTGRES_USER = bla
POSTGRES_PASSWORD = bla
POSTGRES_DB = bubu
POSTGRES_HOST = localhost
POSTGRES_HOST_DOCKER = postgres
POSTGRES_HOST_DOCKER = postgres
SEGMENT_KEY = Etl4WJwzOkeDPAjaOXOMgyU16hO7mV7B

39
level_3/poetry.lock generated
View file

@ -2147,13 +2147,13 @@ six = "*"
[[package]]
name = "langsmith"
version = "0.0.53"
version = "0.0.54"
description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform."
optional = false
python-versions = ">=3.8.1,<4.0"
files = [
{file = "langsmith-0.0.53-py3-none-any.whl", hash = "sha256:a090b1c7d7968fb8d2476ddd608a5171f0e812a82b1bca29ca136cdea375a74e"},
{file = "langsmith-0.0.53.tar.gz", hash = "sha256:a426a1d39843207a5dd3d72787b5304376541eb818509ee7909bbb696b072488"},
{file = "langsmith-0.0.54-py3-none-any.whl", hash = "sha256:55eca5967cadb661a49ad32aecda48a824fadef202ca384575209a9d6f823b74"},
{file = "langsmith-0.0.54.tar.gz", hash = "sha256:76c8e34b4d10ad93541107138089635829f9d60601a7f6bddf5ba582d178e521"},
]
[package.dependencies]
@ -2576,6 +2576,17 @@ files = [
{file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
]
[[package]]
name = "monotonic"
version = "1.6"
description = "An implementation of time.monotonic() for Python 2 & < 3.3"
optional = false
python-versions = "*"
files = [
{file = "monotonic-1.6-py2.py3-none-any.whl", hash = "sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c"},
{file = "monotonic-1.6.tar.gz", hash = "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7"},
]
[[package]]
name = "mpmath"
version = "1.3.0"
@ -4572,6 +4583,26 @@ dev = ["flake8", "mypy", "pycodestyle", "typing_extensions"]
doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-panels (>=0.5.2)", "sphinx-tabs"]
test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "scikit-umfpack", "threadpoolctl"]
[[package]]
name = "segment-analytics-python"
version = "2.2.3"
description = "The hassle-free way to integrate analytics into any python application."
optional = false
python-versions = ">=3.6.0"
files = [
{file = "segment-analytics-python-2.2.3.tar.gz", hash = "sha256:0df5908e3df74b4482f33392fdd450df4c8351bf54974376fbe6bf33b0700865"},
{file = "segment_analytics_python-2.2.3-py2.py3-none-any.whl", hash = "sha256:06cc3d8e79103f02c3878ec66cb66152415473d0d2a142b98a0ee18da972e109"},
]
[package.dependencies]
backoff = ">=2.1,<3.0"
monotonic = ">=1.5,<2.0"
python-dateutil = ">=2.2,<3.0"
requests = ">=2.7,<3.0"
[package.extras]
test = ["flake8 (==3.7.9)", "mock (==2.0.0)", "pylint (==2.8.0)"]
[[package]]
name = "semver"
version = "3.0.2"
@ -6209,4 +6240,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
[metadata]
lock-version = "2.0"
python-versions = "^3.10"
content-hash = "9a121700fa6a5d396844af0958e72b437b5819cf30970f4176e002709feca667"
content-hash = "1700f1cab54ed107d299a47031ad53c58e5e72e3791b9113652fb4ff7854a91a"

1
level_3/pyproject.toml
View file

@ -40,6 +40,7 @@ dash = "^2.14.0"
unstructured = {extras = ["pdf"], version = "^0.10.23"}
sentence-transformers = "2.2.2"
torch = "2.0.*"
segment-analytics-python = "^2.2.3"

98
level_3/rag_test_manager.py
View file

@ -32,6 +32,8 @@ from models.metadatas import MetaDatas
from models.operation import Operation
from models.docs import DocsModel
import segment.analytics as analytics
load_dotenv()
import ast
import tracemalloc
@ -55,8 +57,16 @@ from database.database_crud import session_scope
from database.database import AsyncSessionLocal
openai.api_key = os.getenv("OPENAI_API_KEY", "")
analytics.write_key = os.getenv("SEGMENT_KEY", "")
def on_error(error, items):
print("An error occurred:", error)
analytics.debug = True
analytics.on_error = on_error
async def retrieve_latest_test_case(session, user_id, memory_id):
try:
# Use await with session.execute() and row.fetchone() or row.all() for async query execution
@ -476,6 +486,10 @@ async def start_test(
await add_entity(
session, TestSet(id=test_set_id, user_id=user_id, content=str(test_set))
)
analytics.track(user_id, 'TestSet', {
'id': test_set_id,
'content': str(test_set)
})
if params is None:
data_format = data_format_route(
@ -521,6 +535,15 @@ async def start_test(
test_set_id=test_set_id,
),
)
analytics.track(user_id, 'Operation', {
'id': job_id,
'operation_params': str(test_params),
'number_of_files': count_files_in_data_folder(),
'operation_status': "RUNNING",
'operation_type': retriever_type,
'test_set_id': test_set_id,
})
doc_names = get_document_names(data)
for doc in doc_names:
@ -697,6 +720,20 @@ async def start_test(
test_params=str(chunk), # Add params to the database table
),
)
analytics.track(user_id, 'TestOutput', {
'test_set_id': test_set_id,
'operation_id': job_id,
'set_id' : str(uuid.uuid4()),
'test_results' : result["success"],
'test_score' : str(result["score"]),
'test_metric_name' : result["metric_name"],
'test_query' : result["query"],
'test_output' : result["output"],
'test_expected_output' : str(["expected_output"]),
'test_context' : result["context"][0],
'test_params' : str(chunk),
})
analytics.flush()
await update_entity(session, Operation, job_id, "COMPLETED")
@ -704,49 +741,6 @@ async def start_test(
async def main():
# metadata = {
# "version": "1.0",
# "agreement_id": "AG123456",
# "privacy_policy": "https://example.com/privacy",
# "terms_of_service": "https://example.com/terms",
# "format": "json",
# "schema_version": "1.1",
# "checksum": "a1b2c3d4e5f6",
# "owner": "John Doe",
# "license": "MIT",
# "validity_start": "2023-08-01",
# "validity_end": "2024-07-31",
# }
#
# test_set = [
# {
# "question": "Who is the main character in 'The Call of the Wild'?",
# "answer": "Buck",
# },
# {"question": "Who wrote 'The Call of the Wild'?", "answer": "Jack London"},
# {
# "question": "Where does Buck live at the start of the book?",
# "answer": "In the Santa Clara Valley, at Judge Miller’s place.",
# },
# {
# "question": "Why is Buck kidnapped?",
# "answer": "He is kidnapped to be sold as a sled dog in the Yukon during the Klondike Gold Rush.",
# },
# {
# "question": "How does Buck become the leader of the sled dog team?",
# "answer": "Buck becomes the leader after defeating the original leader, Spitz, in a fight.",
# },
# ]
# "https://www.ibiblio.org/ebooks/London/Call%20of%20Wild.pdf"
# # http://public-library.uk/ebooks/59/83.pdf
# result = await start_test(
# [".data/3ZCCCW.pdf"],
# test_set=test_set,
# user_id="677",
# params=["chunk_size", "search_type"],
# metadata=metadata,
# retriever_type="single_document_context",
# )
parser = argparse.ArgumentParser(description="Run tests against a document.")
parser.add_argument("--file", nargs="+", required=True, help="List of file paths to test.")
@ -793,21 +787,3 @@ async def main():
if __name__ == "__main__":
asyncio.run(main())
# delete_mems = await memory.dynamic_method_call(dynamic_memory_class, 'delete_memories',
# namespace=test_id)
# test_load_pipeline = await asyncio.gather(
# *(run_load_test_element(test_item,loader_settings, metadata, test_id) for test_item in test_set)
# )
#
# test_eval_pipeline = await asyncio.gather(
# *(run_search_eval_element(test_item, test_id) for test_item in test_set)
# )
# logging.info("Results of the eval pipeline %s", str(test_eval_pipeline))
# await add_entity(session, TestOutput(id=test_id, user_id=user_id, test_results=str(test_eval_pipeline)))
# return test_eval_pipeline
# # Gather and run all tests in parallel
# results = await asyncio.gather(
# *(run_testo(test, loader_settings, metadata) for test in test_params)
# )
# return results