Update docs, fix issue with params
This commit is contained in:
Vasilije
parent
57ca73ca47
commit
552a8e65f9
1 changed files with 195 additions and 195 deletions
|
|
@ -506,201 +506,201 @@ async def start_test(
|
||||||
"source": f"{data_location}",
|
"source": f"{data_location}",
|
||||||
"path": data,
|
"path": data,
|
||||||
}
|
}
|
||||||
# if job_id is None:
|
if job_id is None:
|
||||||
# job_id = str(uuid.uuid4())
|
job_id = str(uuid.uuid4())
|
||||||
#
|
|
||||||
# await add_entity(
|
await add_entity(
|
||||||
# session,
|
session,
|
||||||
# Operation(
|
Operation(
|
||||||
# id=job_id,
|
id=job_id,
|
||||||
# user_id=user_id,
|
user_id=user_id,
|
||||||
# operation_params=str(test_params),
|
operation_params=str(test_params),
|
||||||
# number_of_files=count_files_in_data_folder(),
|
number_of_files=count_files_in_data_folder(),
|
||||||
# operation_status = "RUNNING",
|
operation_status = "RUNNING",
|
||||||
# operation_type=retriever_type,
|
operation_type=retriever_type,
|
||||||
# test_set_id=test_set_id,
|
test_set_id=test_set_id,
|
||||||
# ),
|
),
|
||||||
# )
|
)
|
||||||
# doc_names = get_document_names(data)
|
doc_names = get_document_names(data)
|
||||||
# for doc in doc_names:
|
for doc in doc_names:
|
||||||
#
|
|
||||||
# await add_entity(
|
await add_entity(
|
||||||
# session,
|
session,
|
||||||
# DocsModel(
|
DocsModel(
|
||||||
# id=str(uuid.uuid4()),
|
id=str(uuid.uuid4()),
|
||||||
# operation_id=job_id,
|
operation_id=job_id,
|
||||||
# doc_name = doc
|
doc_name = doc
|
||||||
# )
|
)
|
||||||
# )
|
)
|
||||||
#
|
|
||||||
# async def run_test(
|
async def run_test(
|
||||||
# test, loader_settings, metadata, test_id=None, retriever_type=False
|
test, loader_settings, metadata, test_id=None, retriever_type=False
|
||||||
# ):
|
):
|
||||||
# if test_id is None:
|
if test_id is None:
|
||||||
# test_id = str(generate_letter_uuid()) + "_" + "SEMANTICMEMORY"
|
test_id = str(generate_letter_uuid()) + "_" + "SEMANTICMEMORY"
|
||||||
# await memory.manage_memory_attributes(existing_user)
|
await memory.manage_memory_attributes(existing_user)
|
||||||
# test_class = test_id + "_class"
|
test_class = test_id + "_class"
|
||||||
|
await memory.add_dynamic_memory_class(test_id.lower(), test_id)
|
||||||
|
dynamic_memory_class = getattr(memory, test_class.lower(), None)
|
||||||
|
methods_to_add = ["add_memories", "fetch_memories", "delete_memories"]
|
||||||
|
|
||||||
|
if dynamic_memory_class is not None:
|
||||||
|
for method_name in methods_to_add:
|
||||||
|
await memory.add_method_to_class(dynamic_memory_class, method_name)
|
||||||
|
print(f"Memory method {method_name} has been added")
|
||||||
|
else:
|
||||||
|
print(f"No attribute named {test_class.lower()} in memory.")
|
||||||
|
|
||||||
|
print(f"Trying to access: {test_class.lower()}")
|
||||||
|
print("Available memory classes:", await memory.list_memory_classes())
|
||||||
|
if test:
|
||||||
|
loader_settings.update(test)
|
||||||
|
# Check if the search_type is 'none'
|
||||||
|
if loader_settings.get('search_type') == 'none':
|
||||||
|
# Change it to 'hybrid'
|
||||||
|
loader_settings['search_type'] = 'hybrid'
|
||||||
|
|
||||||
|
test_class = test_id + "_class"
|
||||||
|
dynamic_memory_class = getattr(memory, test_class.lower(), None)
|
||||||
|
|
||||||
|
async def run_load_test_element(
|
||||||
|
loader_settings=loader_settings,
|
||||||
|
metadata=metadata,
|
||||||
|
test_id=test_id,
|
||||||
|
test_set=test_set,
|
||||||
|
):
|
||||||
|
print(f"Trying to access: {test_class.lower()}")
|
||||||
|
await memory.dynamic_method_call(
|
||||||
|
dynamic_memory_class,
|
||||||
|
"add_memories",
|
||||||
|
observation="Observation loaded",
|
||||||
|
params=metadata,
|
||||||
|
loader_settings=loader_settings,
|
||||||
|
)
|
||||||
|
return "Loaded test element"
|
||||||
|
|
||||||
|
async def run_search_element(test_item, test_id, search_type="text"):
|
||||||
|
retrieve_action = await memory.dynamic_method_call(
|
||||||
|
dynamic_memory_class,
|
||||||
|
"fetch_memories",
|
||||||
|
observation=str(test_item["question"]), search_type=loader_settings.get('search_type'),
|
||||||
|
)
|
||||||
|
print(
|
||||||
|
"Here is the test result",
|
||||||
|
str(retrieve_action),
|
||||||
|
)
|
||||||
|
if loader_settings.get('search_type') == 'bm25':
|
||||||
|
return retrieve_action["data"]["Get"][test_id]
|
||||||
|
else:
|
||||||
|
return retrieve_action["data"]["Get"][test_id][0]["text"]
|
||||||
|
|
||||||
|
async def run_eval(test_item, search_result):
|
||||||
|
logging.info("Initiated test set evaluation")
|
||||||
|
test_eval = await eval_test(
|
||||||
|
query=str(test_item["question"]),
|
||||||
|
expected_output=str(test_item["answer"]),
|
||||||
|
context=str(search_result),
|
||||||
|
)
|
||||||
|
logging.info("Successfully evaluated test set")
|
||||||
|
return test_eval
|
||||||
|
|
||||||
|
async def run_generate_test_set(test_id):
|
||||||
|
test_class = test_id + "_class"
|
||||||
# await memory.add_dynamic_memory_class(test_id.lower(), test_id)
|
# await memory.add_dynamic_memory_class(test_id.lower(), test_id)
|
||||||
# dynamic_memory_class = getattr(memory, test_class.lower(), None)
|
dynamic_memory_class = getattr(memory, test_class.lower(), None)
|
||||||
# methods_to_add = ["add_memories", "fetch_memories", "delete_memories"]
|
print(dynamic_memory_class)
|
||||||
#
|
retrieve_action = await memory.dynamic_method_call(
|
||||||
# if dynamic_memory_class is not None:
|
dynamic_memory_class,
|
||||||
# for method_name in methods_to_add:
|
"fetch_memories",
|
||||||
# await memory.add_method_to_class(dynamic_memory_class, method_name)
|
observation="Generate a short summary of this document",
|
||||||
# print(f"Memory method {method_name} has been added")
|
search_type="generative",
|
||||||
# else:
|
)
|
||||||
# print(f"No attribute named {test_class.lower()} in memory.")
|
return dynamic_test_manager(retrieve_action)
|
||||||
#
|
|
||||||
# print(f"Trying to access: {test_class.lower()}")
|
test_eval_pipeline = []
|
||||||
# print("Available memory classes:", await memory.list_memory_classes())
|
if retriever_type == "llm_context":
|
||||||
# if test:
|
for test_qa in test_set:
|
||||||
# loader_settings.update(test)
|
context = ""
|
||||||
# # Check if the search_type is 'none'
|
logging.info("Loading and evaluating test set for LLM context")
|
||||||
# if loader_settings.get('search_type') == 'none':
|
test_result = await run_eval(test_qa, context)
|
||||||
# # Change it to 'hybrid'
|
test_eval_pipeline.append(test_result)
|
||||||
# loader_settings['search_type'] = 'hybrid'
|
elif retriever_type == "single_document_context":
|
||||||
#
|
if test_set:
|
||||||
# test_class = test_id + "_class"
|
logging.info(
|
||||||
# dynamic_memory_class = getattr(memory, test_class.lower(), None)
|
"Loading and evaluating test set for a single document context"
|
||||||
#
|
)
|
||||||
# async def run_load_test_element(
|
await run_load_test_element(
|
||||||
# loader_settings=loader_settings,
|
loader_settings, metadata, test_id, test_set
|
||||||
# metadata=metadata,
|
)
|
||||||
# test_id=test_id,
|
for test_qa in test_set:
|
||||||
# test_set=test_set,
|
result = await run_search_element(test_qa, test_id)
|
||||||
# ):
|
test_result = await run_eval(test_qa, result)
|
||||||
# print(f"Trying to access: {test_class.lower()}")
|
test_result.append(test)
|
||||||
# await memory.dynamic_method_call(
|
test_eval_pipeline.append(test_result)
|
||||||
# dynamic_memory_class,
|
await memory.dynamic_method_call(
|
||||||
# "add_memories",
|
dynamic_memory_class, "delete_memories", namespace=test_id
|
||||||
# observation="Observation loaded",
|
)
|
||||||
# params=metadata,
|
else:
|
||||||
# loader_settings=loader_settings,
|
pass
|
||||||
# )
|
if generate_test_set is True:
|
||||||
# return "Loaded test element"
|
synthetic_test_set = run_generate_test_set(test_id)
|
||||||
#
|
else:
|
||||||
# async def run_search_element(test_item, test_id, search_type="text"):
|
pass
|
||||||
# retrieve_action = await memory.dynamic_method_call(
|
|
||||||
# dynamic_memory_class,
|
return test_id, test_eval_pipeline
|
||||||
# "fetch_memories",
|
|
||||||
# observation=str(test_item["question"]), search_type=loader_settings.get('search_type'),
|
results = []
|
||||||
# )
|
|
||||||
# print(
|
logging.info("Validating the retriever type")
|
||||||
# "Here is the test result",
|
|
||||||
# str(retrieve_action),
|
logging.info("Retriever type: %s", retriever_type)
|
||||||
# )
|
|
||||||
# if loader_settings.get('search_type') == 'bm25':
|
if retriever_type == "llm_context":
|
||||||
# return retrieve_action["data"]["Get"][test_id]
|
logging.info("Retriever type: llm_context")
|
||||||
# else:
|
test_id, result = await run_test(
|
||||||
# return retrieve_action["data"]["Get"][test_id][0]["text"]
|
test=None,
|
||||||
#
|
loader_settings=loader_settings,
|
||||||
# async def run_eval(test_item, search_result):
|
metadata=metadata,
|
||||||
# logging.info("Initiated test set evaluation")
|
retriever_type=retriever_type,
|
||||||
# test_eval = await eval_test(
|
) # No params for this case
|
||||||
# query=str(test_item["question"]),
|
results.append([result, "No params"])
|
||||||
# expected_output=str(test_item["answer"]),
|
|
||||||
# context=str(search_result),
|
elif retriever_type == "single_document_context":
|
||||||
# )
|
logging.info("Retriever type: single document context")
|
||||||
# logging.info("Successfully evaluated test set")
|
for param in test_params:
|
||||||
# return test_eval
|
logging.info("Running for chunk size %s", param["chunk_size"])
|
||||||
#
|
test_id, result = await run_test(
|
||||||
# async def run_generate_test_set(test_id):
|
param, loader_settings, metadata, retriever_type=retriever_type
|
||||||
# test_class = test_id + "_class"
|
) # Add the params to the result
|
||||||
# # await memory.add_dynamic_memory_class(test_id.lower(), test_id)
|
# result.append(param)
|
||||||
# dynamic_memory_class = getattr(memory, test_class.lower(), None)
|
results.append(result)
|
||||||
# print(dynamic_memory_class)
|
|
||||||
# retrieve_action = await memory.dynamic_method_call(
|
for b in results:
|
||||||
# dynamic_memory_class,
|
logging.info("Loading %s", str(b))
|
||||||
# "fetch_memories",
|
for result, chunk in b:
|
||||||
# observation="Generate a short summary of this document",
|
logging.info("Loading %s", str(result))
|
||||||
# search_type="generative",
|
await add_entity(
|
||||||
# )
|
session,
|
||||||
# return dynamic_test_manager(retrieve_action)
|
TestOutput(
|
||||||
#
|
id=test_id,
|
||||||
# test_eval_pipeline = []
|
test_set_id=test_set_id,
|
||||||
# if retriever_type == "llm_context":
|
operation_id=job_id,
|
||||||
# for test_qa in test_set:
|
set_id=str(uuid.uuid4()),
|
||||||
# context = ""
|
user_id=user_id,
|
||||||
# logging.info("Loading and evaluating test set for LLM context")
|
test_results=result["success"],
|
||||||
# test_result = await run_eval(test_qa, context)
|
test_score=str(result["score"]),
|
||||||
# test_eval_pipeline.append(test_result)
|
test_metric_name=result["metric_name"],
|
||||||
# elif retriever_type == "single_document_context":
|
test_query=result["query"],
|
||||||
# if test_set:
|
test_output=result["output"],
|
||||||
# logging.info(
|
test_expected_output=str(["expected_output"]),
|
||||||
# "Loading and evaluating test set for a single document context"
|
test_context=result["context"][0],
|
||||||
# )
|
test_params=str(chunk), # Add params to the database table
|
||||||
# await run_load_test_element(
|
),
|
||||||
# loader_settings, metadata, test_id, test_set
|
)
|
||||||
# )
|
|
||||||
# for test_qa in test_set:
|
await update_entity(session, Operation, job_id, "COMPLETED")
|
||||||
# result = await run_search_element(test_qa, test_id)
|
|
||||||
# test_result = await run_eval(test_qa, result)
|
return results
|
||||||
# test_result.append(test)
|
|
||||||
# test_eval_pipeline.append(test_result)
|
|
||||||
# await memory.dynamic_method_call(
|
|
||||||
# dynamic_memory_class, "delete_memories", namespace=test_id
|
|
||||||
# )
|
|
||||||
# else:
|
|
||||||
# pass
|
|
||||||
# if generate_test_set is True:
|
|
||||||
# synthetic_test_set = run_generate_test_set(test_id)
|
|
||||||
# else:
|
|
||||||
# pass
|
|
||||||
#
|
|
||||||
# return test_id, test_eval_pipeline
|
|
||||||
#
|
|
||||||
# results = []
|
|
||||||
#
|
|
||||||
# logging.info("Validating the retriever type")
|
|
||||||
#
|
|
||||||
# logging.info("Retriever type: %s", retriever_type)
|
|
||||||
#
|
|
||||||
# if retriever_type == "llm_context":
|
|
||||||
# logging.info("Retriever type: llm_context")
|
|
||||||
# test_id, result = await run_test(
|
|
||||||
# test=None,
|
|
||||||
# loader_settings=loader_settings,
|
|
||||||
# metadata=metadata,
|
|
||||||
# retriever_type=retriever_type,
|
|
||||||
# ) # No params for this case
|
|
||||||
# results.append([result, "No params"])
|
|
||||||
#
|
|
||||||
# elif retriever_type == "single_document_context":
|
|
||||||
# logging.info("Retriever type: single document context")
|
|
||||||
# for param in test_params:
|
|
||||||
# logging.info("Running for chunk size %s", param["chunk_size"])
|
|
||||||
# test_id, result = await run_test(
|
|
||||||
# param, loader_settings, metadata, retriever_type=retriever_type
|
|
||||||
# ) # Add the params to the result
|
|
||||||
# # result.append(param)
|
|
||||||
# results.append(result)
|
|
||||||
#
|
|
||||||
# for b in results:
|
|
||||||
# logging.info("Loading %s", str(b))
|
|
||||||
# for result, chunk in b:
|
|
||||||
# logging.info("Loading %s", str(result))
|
|
||||||
# await add_entity(
|
|
||||||
# session,
|
|
||||||
# TestOutput(
|
|
||||||
# id=test_id,
|
|
||||||
# test_set_id=test_set_id,
|
|
||||||
# operation_id=job_id,
|
|
||||||
# set_id=str(uuid.uuid4()),
|
|
||||||
# user_id=user_id,
|
|
||||||
# test_results=result["success"],
|
|
||||||
# test_score=str(result["score"]),
|
|
||||||
# test_metric_name=result["metric_name"],
|
|
||||||
# test_query=result["query"],
|
|
||||||
# test_output=result["output"],
|
|
||||||
# test_expected_output=str(["expected_output"]),
|
|
||||||
# test_context=result["context"][0],
|
|
||||||
# test_params=str(chunk), # Add params to the database table
|
|
||||||
# ),
|
|
||||||
# )
|
|
||||||
#
|
|
||||||
# await update_entity(session, Operation, job_id, "COMPLETED")
|
|
||||||
#
|
|
||||||
# return results
|
|
||||||
|
|
||||||
|
|
||||||
async def main():
|
async def main():
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue