From 75aded58b7fd1125f650b75013683403c14c1441 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Sun, 15 Oct 2023 21:25:43 +0200 Subject: [PATCH] Fixed script, added base LLM eval option --- level_3/rag_test_manager.py | 159 ++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 79 deletions(-) diff --git a/level_3/rag_test_manager.py b/level_3/rag_test_manager.py index a75eb6c11..a584b3c02 100644 --- a/level_3/rag_test_manager.py +++ b/level_3/rag_test_manager.py @@ -1,4 +1,5 @@ - +import argparse +import json import random import itertools @@ -391,86 +392,86 @@ async def start_test(data, test_set=None, user_id=None, params=None, job_id=None async def main(): - metadata = { - "version": "1.0", - "agreement_id": "AG123456", - "privacy_policy": "https://example.com/privacy", - "terms_of_service": "https://example.com/terms", - "format": "json", - "schema_version": "1.1", - "checksum": "a1b2c3d4e5f6", - "owner": "John Doe", - "license": "MIT", - "validity_start": "2023-08-01", - "validity_end": "2024-07-31", - } + # metadata = { + # "version": "1.0", + # "agreement_id": "AG123456", + # "privacy_policy": "https://example.com/privacy", + # "terms_of_service": "https://example.com/terms", + # "format": "json", + # "schema_version": "1.1", + # "checksum": "a1b2c3d4e5f6", + # "owner": "John Doe", + # "license": "MIT", + # "validity_start": "2023-08-01", + # "validity_end": "2024-07-31", + # } + # + # test_set = [ + # { + # "question": "Who is the main character in 'The Call of the Wild'?", + # "answer": "Buck" + # }, + # { + # "question": "Who wrote 'The Call of the Wild'?", + # "answer": "Jack London" + # }, + # { + # "question": "Where does Buck live at the start of the book?", + # "answer": "In the Santa Clara Valley, at Judge Miller’s place." + # }, + # { + # "question": "Why is Buck kidnapped?", + # "answer": "He is kidnapped to be sold as a sled dog in the Yukon during the Klondike Gold Rush." + # }, + # { + # "question": "How does Buck become the leader of the sled dog team?", + # "answer": "Buck becomes the leader after defeating the original leader, Spitz, in a fight." + # } + # ] + # # "https://www.ibiblio.org/ebooks/London/Call%20of%20Wild.pdf" + # #http://public-library.uk/ebooks/59/83.pdf + # result = await start_test("http://public-library.uk/ebooks/59/83.pdf", test_set=test_set, user_id="676", params=None, metadata=metadata) + # # + parser = argparse.ArgumentParser(description="Run tests against a document.") + parser.add_argument("--url", required=True, help="URL of the document to test.") + parser.add_argument("--test_set", required=True, help="Path to JSON file containing the test set.") + parser.add_argument("--user_id", required=True, help="User ID.") + parser.add_argument("--params", help="Additional parameters in JSON format.") + parser.add_argument("--metadata", required=True, help="Path to JSON file containing metadata.") + parser.add_argument("--generate_test_set", required=True, help="Make a test set.") + parser.add_argument("--only_llm_context", required=True, help="Do a test only within the existing LLM context") + args = parser.parse_args() - test_set = [ - { - "question": "Who is the main character in 'The Call of the Wild'?", - "answer": "Buck" - }, - { - "question": "Who wrote 'The Call of the Wild'?", - "answer": "Jack London" - }, - { - "question": "Where does Buck live at the start of the book?", - "answer": "In the Santa Clara Valley, at Judge Miller’s place." - }, - { - "question": "Why is Buck kidnapped?", - "answer": "He is kidnapped to be sold as a sled dog in the Yukon during the Klondike Gold Rush." - }, - { - "question": "How does Buck become the leader of the sled dog team?", - "answer": "Buck becomes the leader after defeating the original leader, Spitz, in a fight." - } - ] - # "https://www.ibiblio.org/ebooks/London/Call%20of%20Wild.pdf" - #http://public-library.uk/ebooks/59/83.pdf - result = await start_test("http://public-library.uk/ebooks/59/83.pdf", test_set=test_set, user_id="676", params=None, metadata=metadata) - # - # parser = argparse.ArgumentParser(description="Run tests against a document.") - # parser.add_argument("--url", required=True, help="URL of the document to test.") - # parser.add_argument("--test_set", required=True, help="Path to JSON file containing the test set.") - # parser.add_argument("--user_id", required=True, help="User ID.") - # parser.add_argument("--params", help="Additional parameters in JSON format.") - # parser.add_argument("--metadata", required=True, help="Path to JSON file containing metadata.") - # parser.add_argument("--generate_test_set", required=True, help="Make a test set.") - # parser.add_argument("--only_llm_context", required=True, help="Do a test only within the existing LLM context") - # args = parser.parse_args() + try: + with open(args.test_set, "r") as file: + test_set = json.load(file) + if not isinstance(test_set, list): # Expecting a list + raise TypeError("Parsed test_set JSON is not a list.") + except Exception as e: + print(f"Error loading test_set: {str(e)}") + return - # try: - # with open(args.test_set, "r") as file: - # test_set = json.load(file) - # if not isinstance(test_set, list): # Expecting a list - # raise TypeError("Parsed test_set JSON is not a list.") - # except Exception as e: - # print(f"Error loading test_set: {str(e)}") - # return - # - # try: - # with open(args.metadata, "r") as file: - # metadata = json.load(file) - # if not isinstance(metadata, dict): - # raise TypeError("Parsed metadata JSON is not a dictionary.") - # except Exception as e: - # print(f"Error loading metadata: {str(e)}") - # return - # - # if args.params: - # try: - # params = json.loads(args.params) - # if not isinstance(params, dict): - # raise TypeError("Parsed params JSON is not a dictionary.") - # except json.JSONDecodeError as e: - # print(f"Error parsing params: {str(e)}") - # return - # else: - # params = None - # #clean up params here - # await start_test(args.url, test_set, args.user_id, params=None, metadata=metadata) + try: + with open(args.metadata, "r") as file: + metadata = json.load(file) + if not isinstance(metadata, dict): + raise TypeError("Parsed metadata JSON is not a dictionary.") + except Exception as e: + print(f"Error loading metadata: {str(e)}") + return + + if args.params: + try: + params = json.loads(args.params) + if not isinstance(params, dict): + raise TypeError("Parsed params JSON is not a dictionary.") + except json.JSONDecodeError as e: + print(f"Error parsing params: {str(e)}") + return + else: + params = None + #clean up params here + await start_test(args.url, test_set, args.user_id, params=None, metadata=metadata) if __name__ == "__main__": asyncio.run(main())