From 343de01d5af23c1c7f763a58e417d7852e73c2d6 Mon Sep 17 00:00:00 2001 From: hande-k Date: Thu, 23 Jan 2025 11:11:51 +0100 Subject: [PATCH] update notebooks with latest eval --- notebooks/cognee_demo.ipynb | 955 +++++++++-------------- notebooks/cognee_hotpot_eval.ipynb | 1169 +++++++++++++++++++++++++--- notebooks/hr_demo.ipynb | 373 +++++++-- 3 files changed, 1745 insertions(+), 752 deletions(-) diff --git a/notebooks/cognee_demo.ipynb b/notebooks/cognee_demo.ipynb index f59f6338c..6540d19c5 100644 --- a/notebooks/cognee_demo.ipynb +++ b/notebooks/cognee_demo.ipynb @@ -265,6 +265,7 @@ }, { "cell_type": "code", + "execution_count": null, "id": "df16431d0f48b006", "metadata": { "ExecuteTime": { @@ -272,6 +273,7 @@ "start_time": "2024-12-24T11:53:59.347420Z" } }, + "outputs": [], "source": [ "job_position = \"\"\"Senior Data Scientist (Machine Learning)\n", "\n", @@ -298,12 +300,11 @@ "Strong problem-solving skills and attention to detail.\n", "Candidate CVs\n", "\"\"\"\n" - ], - "outputs": [], - "execution_count": 1 + ] }, { "cell_type": "code", + "execution_count": null, "id": "9086abf3af077ab4", "metadata": { "ExecuteTime": { @@ -311,6 +312,7 @@ "start_time": "2024-12-24T11:53:59.363662Z" } }, + "outputs": [], "source": [ "job_1 = \"\"\"\n", "CV 1: Relevant\n", @@ -343,12 +345,11 @@ "Big Data Technologies: Hadoop, Spark\n", "Data Visualization: Tableau, Matplotlib\n", "\"\"\"" - ], - "outputs": [], - "execution_count": 2 + ] }, { "cell_type": "code", + "execution_count": null, "id": "a9de0cc07f798b7f", "metadata": { "ExecuteTime": { @@ -356,6 +357,7 @@ "start_time": "2024-12-24T11:53:59.371152Z" } }, + "outputs": [], "source": [ "job_2 = \"\"\"\n", "CV 2: Relevant\n", @@ -387,12 +389,11 @@ "Data Visualization: Seaborn, Plotly\n", "Databases: MySQL, MongoDB\n", "\"\"\"" - ], - "outputs": [], - "execution_count": 3 + ] }, { "cell_type": "code", + "execution_count": null, "id": "185ff1c102d06111", "metadata": { "ExecuteTime": { @@ -400,6 +401,7 @@ "start_time": "2024-12-24T11:53:59.959103Z" } }, + "outputs": [], "source": [ "job_3 = \"\"\"\n", "CV 3: Relevant\n", @@ -431,12 +433,11 @@ "Statistical Analysis: SAS, SPSS\n", "Cloud Platforms: AWS, Azure\n", "\"\"\"" - ], - "outputs": [], - "execution_count": 4 + ] }, { "cell_type": "code", + "execution_count": null, "id": "d55ce4c58f8efb67", "metadata": { "ExecuteTime": { @@ -444,6 +445,7 @@ "start_time": "2024-12-24T11:54:00.654716Z" } }, + "outputs": [], "source": [ "job_4 = \"\"\"\n", "CV 4: Not Relevant\n", @@ -473,12 +475,11 @@ "Web Design: HTML, CSS\n", "Specialties: Branding and Identity, Typography\n", "\"\"\"" - ], - "outputs": [], - "execution_count": 5 + ] }, { "cell_type": "code", + "execution_count": null, "id": "ca4ecc32721ad332", "metadata": { "ExecuteTime": { @@ -486,6 +487,7 @@ "start_time": "2024-12-24T11:54:01.183028Z" } }, + "outputs": [], "source": [ "job_5 = \"\"\"\n", "CV 5: Not Relevant\n", @@ -515,9 +517,7 @@ "CRM Software: Salesforce, Zoho\n", "Negotiation and Relationship Building\n", "\"\"\"" - ], - "outputs": [], - "execution_count": 6 + ] }, { "cell_type": "markdown", @@ -529,6 +529,7 @@ }, { "cell_type": "code", + "execution_count": null, "id": "bce39dc6", "metadata": { "ExecuteTime": { @@ -536,6 +537,7 @@ "start_time": "2024-12-24T11:54:04.414132Z" } }, + "outputs": [], "source": [ "import os\n", "\n", @@ -573,12 +575,11 @@ "# os.environ[\"DB_PORT\"]=\"5432\"\n", "# os.environ[\"DB_USERNAME\"]=\"cognee\"\n", "# os.environ[\"DB_PASSWORD\"]=\"cognee\"" - ], - "outputs": [], - "execution_count": 7 + ] }, { "cell_type": "code", + "execution_count": null, "id": "9f1a1dbd", "metadata": { "ExecuteTime": { @@ -586,6 +587,7 @@ "start_time": "2024-12-24T11:54:07.425202Z" } }, + "outputs": [], "source": [ "# Reset the cognee system with the following command:\n", "\n", @@ -593,32 +595,7 @@ "\n", "await cognee.prune.prune_data()\n", "await cognee.prune.prune_system(metadata=True)" - ], - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "File /Users/vasilije/cognee/cognee/.cognee_system/databases/cognee_graph.pkl not found. Initializing an empty graph." - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Database deleted successfully.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/vasilije/cognee/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "execution_count": 8 + ] }, { "cell_type": "markdown", @@ -630,6 +607,7 @@ }, { "cell_type": "code", + "execution_count": null, "id": "904df61ba484a8e5", "metadata": { "ExecuteTime": { @@ -637,41 +615,12 @@ "start_time": "2024-12-24T11:54:23.756587Z" } }, + "outputs": [], "source": [ "import cognee\n", "\n", "await cognee.add([job_1, job_2, job_3, job_4, job_5, job_position], \"example\")" - ], - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "User df77f15b-a077-4c86-a3e4-c059bf4cacb9 has registered.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/vasilije/cognee/.venv/lib/python3.11/site-packages/dlt/destinations/impl/sqlalchemy/merge_job.py:194: SAWarning: Table 'file_metadata' already exists within the given MetaData - not copying.\n", - " staging_table_obj = table_obj.to_metadata(\n", - "/Users/vasilije/cognee/.venv/lib/python3.11/site-packages/dlt/destinations/impl/sqlalchemy/merge_job.py:229: SAWarning: implicitly coercing SELECT object to scalar subquery; please use the .scalar_subquery() method to produce a scalar subquery.\n", - " order_by=order_dir_func(order_by_col),\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Pipeline file_load_from_filesystem load step completed in 0.03 seconds\n", - "1 load package(s) were loaded to destination sqlalchemy and into dataset main\n", - "The sqlalchemy destination used sqlite:////Users/vasilije/cognee/cognee/.cognee_system/databases/cognee_db location to store data\n", - "Load package 1735041267.4777632 is LOADED and contains no failed jobs\n" - ] - } - ], - "execution_count": 9 + ] }, { "cell_type": "markdown", @@ -683,6 +632,7 @@ }, { "cell_type": "code", + "execution_count": null, "id": "7c431fdef4921ae0", "metadata": { "ExecuteTime": { @@ -690,6 +640,7 @@ "start_time": "2024-12-24T11:54:44.723877Z" } }, + "outputs": [], "source": [ "from cognee.shared.data_models import KnowledgeGraph\n", "from cognee.modules.data.models import Dataset, Data\n", @@ -728,12 +679,11 @@ " print(result)\n", " except Exception as error:\n", " raise error\n" - ], - "outputs": [], - "execution_count": 10 + ] }, { "cell_type": "code", + "execution_count": null, "id": "f0a91b99c6215e09", "metadata": { "ExecuteTime": { @@ -741,6 +691,7 @@ "start_time": "2024-12-24T11:54:47.384342Z" } }, + "outputs": [], "source": [ "from cognee.modules.users.methods import get_default_user\n", "from cognee.modules.data.methods import get_datasets_by_name\n", @@ -750,27 +701,7 @@ "datasets = await get_datasets_by_name([\"example\"], user.id)\n", "\n", "await run_cognify_pipeline(datasets[0], user)" - ], - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "File /Users/vasilije/cognee/cognee/.cognee_system/databases/cognee_graph.pkl not found. Initializing an empty graph./Users/vasilije/cognee/.venv/lib/python3.11/site-packages/pydantic/main.py:1522: RuntimeWarning: fields may not start with an underscore, ignoring \"_metadata\"\n", - " warnings.warn(f'fields may not start with an underscore, ignoring \"{f_name}\"', RuntimeWarning)\n", - "/Users/vasilije/cognee/.venv/lib/python3.11/site-packages/pydantic/main.py:1522: RuntimeWarning: fields may not start with an underscore, ignoring \"__tablename__\"\n", - " warnings.warn(f'fields may not start with an underscore, ignoring \"{f_name}\"', RuntimeWarning)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[TextSummary(id=UUID('92b5d0a7-f980-529d-bb5b-48e72825a01a'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='Experienced Senior Data Scientist with expertise in machine learning and predictive modeling, demonstrating over 8 years in the field.', made_from=DocumentChunk(id=UUID('70b823e2-5b12-57b5-ad8d-798e1d721f8e'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='\\nCV 1: Relevant\\nName: Dr. Emily Carter\\nContact Information:\\n\\nEmail: emily.carter@example.com\\nPhone: (555) 123-4567\\nSummary:\\n\\nSenior Data Scientist with over 8 years of experience in machine learning and predictive analytics. Expertise in developing advanced algorithms and deploying scalable models in production environments.\\n\\nEducation:\\n\\nPh.D. in Computer Science, Stanford University (2014)\\nB.S. in Mathematics, University of California, Berkeley (2010)\\nExperience:\\n\\nSenior Data Scientist, InnovateAI Labs (2016 – Present)\\nLed a team in developing machine learning models for natural language processing applications.\\nImplemented deep learning algorithms that improved prediction accuracy by 25%.\\nCollaborated with cross-functional teams to integrate models into cloud-based platforms.\\nData Scientist, DataWave Analytics (2014 – 2016)\\nDeveloped predictive models for customer segmentation and churn analysis.\\nAnalyzed large datasets using Hadoop and Spark frameworks.\\nSkills:\\n\\nProgramming Languages: Python, R, SQL\\nMachin e Learning: TensorFlow, Keras, Scikit-Learn\\nBig Data Technologies: Hadoop, Spark\\nData Visualization: Tableau, Matplotlib\\n', word_count=133, chunk_index=0, cut_type='sentence_cut', is_part_of=TextDocument(id=UUID('11a2b08a-c160-5961-80b7-b3498eafa973'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='text_85410f4ad1197f5974aef9aed6f103c8', raw_data_location='/Users/vasilije/cognee/cognee/.data_storage/data/text_85410f4ad1197f5974aef9aed6f103c8.txt', metadata_id=UUID('11a2b08a-c160-5961-80b7-b3498eafa973'), mime_type='text/plain', type='text'), contains=[Entity(id=UUID('29e771c8-4c3f-52de-9511-6b705878e130'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='dr. emily carter', is_a=EntityType(id=UUID('d072ba0f-e1a9-58bf-9974-e1802adc8134'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='person', description='person'), description='Senior Data Scientist with over 8 years of experience in machine learning and predictive analytics.'), Entity(id=UUID('ce8b394a-b30e-52fc-b80a-6352edc60e5b'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='stanford university', is_a=EntityType(id=UUID('d3d7b6b4-9b0d-52e8-9e09-a9e9cf4b5a4d'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='organization', description='organization'), description='Prestigious university located in Stanford, California.'), Entity(id=UUID('2c02c93c-9cd1-56b8-9cc0-55ff0b290e57'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='university of california, berkeley', is_a=EntityType(id=UUID('d3d7b6b4-9b0d-52e8-9e09-a9e9cf4b5a4d'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='organization', description='organization'), description='Public research university located in Berkeley, California.'), Entity(id=UUID('9780afb1-dccc-53eb-9a30-c0d4ce033711'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='innovateai labs', is_a=EntityType(id=UUID('d3d7b6b4-9b0d-52e8-9e09-a9e9cf4b5a4d'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='organization', description='organization'), description='A lab focused on artificial intelligence projects.'), Entity(id=UUID('50d0a685-5300-544f-b081-edca4b625886'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='datawave analytics', is_a=EntityType(id=UUID('d3d7b6b4-9b0d-52e8-9e09-a9e9cf4b5a4d'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='organization', description='organization'), description='Analytics firm specialized in data-driven insights.'), Entity(id=UUID('c95db510-e2ee-5a00-bded-20bbcb50c492'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='python', is_a=EntityType(id=UUID('80d409bb-e431-5939-a1ad-3acd96267128'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='programming language', description='programming language'), description='A high-level programming language used for general-purpose programming.'), Entity(id=UUID('39bd9707-8098-52ed-9cbf-bbdd26b963fb'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='r', is_a=EntityType(id=UUID('80d409bb-e431-5939-a1ad-3acd96267128'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='programming language', description='programming language'), description='A programming language and environment for statistical computing and graphics.'), Entity(id=UUID('1ff6821a-b207-5050-83e9-37ff67a27d03'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='sql', is_a=EntityType(id=UUID('80d409bb-e431-5939-a1ad-3acd96267128'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='programming language', description='programming language'), description='A domain-specific language used in programming and managing relational databases.'), Entity(id=UUID('6e72f6f5-0452-5d42-a4e8-4aba6a614cb1'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='tensorflow', is_a=EntityType(id=UUID('9ffe9ce7-8938-5a5c-8d03-5f1a4c5210a1'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='machine learning framework', description='machine learning framework'), description='An open-source software library for dataflow and differentiable programming across a range of tasks.'), Entity(id=UUID('ab85cdff-2a98-5c6d-99a3-df1f40f4ec16'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='keras', is_a=EntityType(id=UUID('9ffe9ce7-8938-5a5c-8d03-5f1a4c5210a1'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='machine learning framework', description='machine learning framework'), description='An open-source neural network library written in Python that runs on top of TensorFlow.'), Entity(id=UUID('37eecdcc-fb56-519c-bc18-d0d3afea0c0d'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='scikit-learn', is_a=EntityType(id=UUID('9ffe9ce7-8938-5a5c-8d03-5f1a4c5210a1'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='machine learning framework', description='machine learning framework'), description='A free software machine learning library for the Python programming language.'), Entity(id=UUID('f9a0eeca-c9ff-53b3-90eb-347254d7d7eb'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='hadoop', is_a=EntityType(id=UUID('7c2287d0-16fc-53dc-86ce-8d8e61c8642c'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='big data technology', description='big data technology'), description='An open-source framework for storing and processing large datasets in a distributed computing environment.'), Entity(id=UUID('46a235af-5ed5-5023-a4ec-c253e3f93031'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='spark', is_a=EntityType(id=UUID('7c2287d0-16fc-53dc-86ce-8d8e61c8642c'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='big data technology', description='big data technology'), description='An open-source unified analytics engine for large-scale data processing.'), Entity(id=UUID('c55004f3-8a6d-5130-b8bd-ed8278daa9a4'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='tableau', is_a=EntityType(id=UUID('674cc5fa-7849-575a-917f-90b7b77f52b3'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='data visualization software', description='data visualization software'), description='A visual analytics platform transforming the way we use data to solve problems.'), Entity(id=UUID('3c7adf8f-ef23-5330-a3fe-6a0b791cee2b'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='matplotlib', is_a=EntityType(id=UUID('3f3619fc-ebd1-50ed-adde-cf94e8bb3c1b'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='data visualization library', description='data visualization library'), description='A plotting library for the Python programming language and its numerical mathematics extension NumPy.')])), TextSummary(id=UUID('2f680bef-2edd-566e-b98c-78d549799e77'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='Senior Data Scientist specializing in Machine Learning at TechNova Solutions', made_from=DocumentChunk(id=UUID('eb6617b8-c78c-519b-b765-1eefc2e3a0d7'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='Senior Data Scientist (Machine Learning)\\n\\nCompany: TechNova Solutions\\nLocation: San Francisco, CA\\n\\nJob Description:\\n\\nTechNova Solutions is seeking a Senior Data Scientist specializing in Machine Learning to join our dynamic analytics team. The ideal candidate will have a strong background in developing and deploying machine learning models, working with large datasets, and translating complex data into actionable insights.\\n\\nResponsibilities:\\n\\nDevelop and implement advanced machine learning algorithms and models.\\nAnalyze large, complex datasets to extract meaningful patterns and insights.\\nCollaborate with cross-functional teams to integrate predictive models into products.\\nStay updated with the latest advancements in machine learning and data science.\\nMentor junior data scientists and provide technical guidance.\\nQualifications:\\n\\nMaster’s or Ph.D. in Data Science, Computer Science, Statistics, or a related field.\\n5+ years of experience in data science and machine learning.\\nProficient in Python, R, and SQL.\\nExpe rience with deep learning frameworks (e.g., TensorFlow, PyTorch).\\nStrong problem-solving skills and attention to detail.\\nCandidate CVs\\n', word_count=153, chunk_index=0, cut_type='sentence_cut', is_part_of=TextDocument(id=UUID('171f3035-4c37-5f7b-97c8-6b222404cc9a'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='text_81a5a96a9a7325d40521ea453778ebe0', raw_data_location='/Users/vasilije/cognee/cognee/.data_storage/data/text_81a5a96a9a7325d40521ea453778ebe0.txt', metadata_id=UUID('171f3035-4c37-5f7b-97c8-6b222404cc9a'), mime_type='text/plain', type='text'), contains=[Entity(id=UUID('453a45c9-14e7-5b73-adb8-55991096fef0'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='technova solutions', is_a=EntityType(id=UUID('a6ed6bf1-fe31-5dfe-8ab4-484691fdf219'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='company', description='company'), description='A technology company specializing in data analytics and machine learning.'), Entity(id=UUID('435dbd37-ab20-503c-9e99-ab8b8a3484e5'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='senior data scientist', is_a=EntityType(id=UUID('524c6bbb-1534-5a51-8068-18dd4ae171eb'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='profession', description='profession'), description='A role focused on advanced data analysis and machine learning.'), Entity(id=UUID('198e2ab8-75e9-5931-97ab-da9a5a8e188c'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='san francisco, ca', is_a=EntityType(id=UUID('19dd7d4d-a966-5ed5-82a0-6ae377761a29'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='location', description='location'), description='A city in California, USA.'), Entity(id=UUID('5187986a-7305-5a63-b057-8f2c097419eb'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='machine learning', is_a=EntityType(id=UUID('0198571b-3e94-50ea-8b9f-19e3a31080c0'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='field', description='field'), description='A subset of artificial intelligence focused on the development of algorithms that enable computers to learn from data.'), Entity(id=UUID('d6545b21-153c-58ba-be47-46e5216521a3'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='data science', is_a=EntityType(id=UUID('0198571b-3e94-50ea-8b9f-19e3a31080c0'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='field', description='field'), description='A multidisciplinary field that uses scientific methods to extract knowledge and insights from data.'), Entity(id=UUID('c0d95499-de6b-5fcf-b0f5-9cbf427ad5c6'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='pytorch', is_a=EntityType(id=UUID('36a32bd3-8880-547a-949b-8447477d1ef5'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='framework', description='framework'), description='An open-source machine learning framework for deep learning.'), Entity(id=UUID('62b4dda1-de4a-5098-a56e-d3fe81f84dbc'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='masters or ph.d. in data science', is_a=EntityType(id=UUID('a49b283a-ce92-50e0-b7fa-ca7c628eb01a'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='degree', description='degree'), description='Advanced academic degree in data science or related fields.')])), TextSummary(id=UUID('5c988618-db52-5979-9cf8-db80c0098285'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='Data Scientist with expertise in machine learning and statistical analysis, adept at managing extensive datasets and converting data into practical business solutions.', made_from=DocumentChunk(id=UUID('a6e82ac7-e791-5d6b-b4a9-f5e41cbe95bf'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='\\nCV 2: Relevant\\nName: Michael Rodriguez\\nContact Information:\\n\\nEmail: michael.rodriguez@example.com\\nPhone: (555) 234-5678\\nSummary:\\n\\nData Scientist with a strong background in machine learning and statistical modeling. Skilled in handling large datasets and translating data into actionable business insights.\\n\\nEducation:\\n\\nM.S. in Data Science, Carnegie Mellon University (2013)\\nB.S. in Computer Science, University of Michigan (2011)\\nExperience:\\n\\nSenior Data Scientist, Alpha Analytics (2017 – Present)\\nDeveloped machine learning models to optimize marketing strategies.\\nReduced customer acquisition cost by 15% through predictive modeling.\\nData Scientist, TechInsights (2013 – 2017)\\nAnalyzed user behavior data to improve product features.\\nImplemented A/B testing frameworks to evaluate product changes.\\nSkills:\\n\\nProgramming Languages: Python, Java, SQL\\nMachine Learning: Scikit-Learn, XGBoost\\nData Visualization: Seaborn, Plotly\\nDatabases: MySQL, MongoDB\\n', word_count=108, chunk_index=0, cut_type='sentence_cut', is_part_of=TextDocument(id=UUID('1f078b0a-3cc1-57a9-9802-f78565d49f29'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='text_f0f63a5c88dbbeef1eca23d32848220c', raw_data_location='/Users/vasilije/cognee/cognee/.data_storage/data/text_f0f63a5c88dbbeef1eca23d32848220c.txt', metadata_id=UUID('1f078b0a-3cc1-57a9-9802-f78565d49f29'), mime_type='text/plain', type='text'), contains=[Entity(id=UUID('73ae630f-7b09-5dce-8c18-45d0a57b30f9'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='michael rodriguez', is_a=EntityType(id=UUID('d072ba0f-e1a9-58bf-9974-e1802adc8134'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='person', description='person'), description='Data Scientist with a strong background in machine learning and statistical modeling.'), Entity(id=UUID('5534e0b0-d0c4-5ab9-82e9-91bed36f70bd'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='carnegie mellon university', is_a=EntityType(id=UUID('912b273c-683d-53ea-8ffe-aadef0b84237'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='educational institution', description='educational institution'), description='University known for its data science program.'), Entity(id=UUID('0af613e0-c11b-550d-ada2-2c2aa6550884'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='university of michigan', is_a=EntityType(id=UUID('912b273c-683d-53ea-8ffe-aadef0b84237'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='educational institution', description='educational institution'), description='University known for its computer science program.'), Entity(id=UUID('04a91fef-8a07-5d50-8f1b-46f3afeec497'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='alpha analytics', is_a=EntityType(id=UUID('a6ed6bf1-fe31-5dfe-8ab4-484691fdf219'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='company', description='company'), description='Company where Michael Rodriguez works as a Senior Data Scientist.'), Entity(id=UUID('3f848ed6-902f-5a8e-9577-cb67f8c17acd'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='techinsights', is_a=EntityType(id=UUID('a6ed6bf1-fe31-5dfe-8ab4-484691fdf219'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='company', description='company'), description='Company where Michael Rodriguez worked as a Data Scientist.')])), TextSummary(id=UUID('ee6cb607-27eb-5b87-bf2a-305721534263'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='Sales Manager with proven ability to enhance revenue and cultivate effective teams. Strong communicator and leader.', made_from=DocumentChunk(id=UUID('7e35407f-7c59-5429-8824-23f1d17118c0'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text=\"\\nCV 5: Not Relevant\\nName: Jessica Miller\\nContact Information:\\n\\nEmail: jessica.miller@example.com\\nPhone: (555) 567-8901\\nSummary:\\n\\nExperienced Sales Manager with a strong track record in driving sales growth and building high-performing teams. Excellent communication and leadership skills.\\n\\nEducation:\\n\\nB.A. in Business Administration, University of Southern California (2010)\\nExperience:\\n\\nSales Manager, Global Enterprises (2015 – Present)\\nManaged a sales team of 15 members, achieving a 20% increase in annual revenue.\\nDeveloped sales strategies that expanded customer base by 25%.\\nSales Representative, Market Leaders Inc. (2010 – 2015)\\nConsistently exceeded sales targets and received the 'Top Salesperson' award in 2013.\\nSkills:\\n\\nSales Strategy and Planning\\nTeam Leadership and Development\\nCRM Software: Salesforce, Zoho\\nNegotiation and Relationship Building\\n\", word_count=102, chunk_index=0, cut_type='sentence_cut', is_part_of=TextDocument(id=UUID('3c323fc9-9165-52da-a079-2627a9556b08'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='text_9b35c7df1f5d4dc84e78270c0bf9cac6', raw_data_location='/Users/vasilije/cognee/cognee/.data_storage/data/text_9b35c7df1f5d4dc84e78270c0bf9cac6.txt', metadata_id=UUID('3c323fc9-9165-52da-a079-2627a9556b08'), mime_type='text/plain', type='text'), contains=[Entity(id=UUID('36a5e3c8-c5f5-5ab5-8d59-ea69d8b36932'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='jessica miller', is_a=EntityType(id=UUID('d072ba0f-e1a9-58bf-9974-e1802adc8134'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='person', description='person'), description='An experienced sales manager with a strong track record in driving sales growth and building high-performing teams.'), Entity(id=UUID('5c32691d-c0e4-5378-9aab-dda8b0fa3931'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='global enterprises', is_a=EntityType(id=UUID('a6ed6bf1-fe31-5dfe-8ab4-484691fdf219'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='company', description='company'), description='A company where Jessica Miller worked as a Sales Manager.'), Entity(id=UUID('67544857-983a-5152-801d-4fc9d35d14e4'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='market leaders inc.', is_a=EntityType(id=UUID('a6ed6bf1-fe31-5dfe-8ab4-484691fdf219'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='company', description='company'), description='A company where Jessica Miller worked as a Sales Representative.'), Entity(id=UUID('f39d6c00-689b-5fd2-9021-893b28ac6ff2'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='university of southern california', is_a=EntityType(id=UUID('912b273c-683d-53ea-8ffe-aadef0b84237'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='educational institution', description='educational institution'), description='University where Jessica Miller obtained her degree in Business Administration.'), Entity(id=UUID('0abc801d-38ca-5003-b974-b60f1956c94a'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='2010', is_a=EntityType(id=UUID('d61d99ac-b291-5666-9748-3e80e1c8b56a'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='date', description='date'), description='Year Jessica Miller graduated from University of Southern California.'), Entity(id=UUID('7c8b43c1-e133-52e6-99aa-239534f1ed45'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='2015', is_a=EntityType(id=UUID('d61d99ac-b291-5666-9748-3e80e1c8b56a'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='date', description='date'), description='Year Jessica Miller started working as Sales Manager at Global Enterprises.'), Entity(id=UUID('2f4749e9-e1e4-5af0-be80-2a10d07557ff'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='present', is_a=EntityType(id=UUID('d61d99ac-b291-5666-9748-3e80e1c8b56a'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='date', description='date'), description=\"Current time indicative of Jessica Miller's ongoing role.\")])), TextSummary(id=UUID('d8a8668e-b122-5713-b289-932407bb294e'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='Creative Graphic Designer with 8+ years of expertise in visual design and branding, skilled in Adobe Creative Suite, dedicated to crafting engaging visuals.', made_from=DocumentChunk(id=UUID('c401b5b1-21d8-5830-8c7b-48e7d94c5b95'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='\\nCV 4: Not Relevant\\nName: David Thompson\\nContact Information:\\n\\nEmail: david.thompson@example.com\\nPhone: (555) 456-7890\\nSummary:\\n\\nCreative Graphic Designer with over 8 years of experience in visual design and branding. Proficient in Adobe Creative Suite and passionate about creating compelling visuals.\\n\\nEducation:\\n\\nB.F.A. in Graphic Design, Rhode Island School of Design (2012)\\nExperience:\\n\\nSenior Graphic Designer, CreativeWorks Agency (2015 – Present)\\nLed design projects for clients in various industries.\\nCreated branding materials that increased client engagement by 30%.\\nGraphic Designer, Visual Innovations (2012 – 2015)\\nDesigned marketing collateral, including brochures, logos, and websites.\\nCollaborated with the marketing team to develop cohesive brand strategies.\\nSkills:\\n\\nDesign Software: Adobe Photoshop, Illustrator, InDesign\\nWeb Design: HTML, CSS\\nSpecialties: Branding and Identity, Typography\\n', word_count=108, chunk_index=0, cut_type='sentence_cut', is_part_of=TextDocument(id=UUID('e71daf63-15a0-50fe-a909-766bc8fd311b'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='text_9abf20fa7defd7e49296c51b4e38edf2', raw_data_location='/Users/vasilije/cognee/cognee/.data_storage/data/text_9abf20fa7defd7e49296c51b4e38edf2.txt', metadata_id=UUID('e71daf63-15a0-50fe-a909-766bc8fd311b'), mime_type='text/plain', type='text'), contains=[Entity(id=UUID('a4777597-06c7-562c-bc44-56f74571a01a'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='david thompson', is_a=EntityType(id=UUID('d072ba0f-e1a9-58bf-9974-e1802adc8134'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='person', description='person'), description='Creative Graphic Designer with over 8 years of experience in visual design and branding.'), Entity(id=UUID('ca20272a-3e88-552f-92fe-491e23f117f8'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='creativeworks agency', is_a=EntityType(id=UUID('d3d7b6b4-9b0d-52e8-9e09-a9e9cf4b5a4d'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='organization', description='organization'), description='An agency where David Thompson is a Senior Graphic Designer.'), Entity(id=UUID('1e97bb97-4d29-5fb8-863a-15ab51f1dd46'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='visual innovations', is_a=EntityType(id=UUID('d3d7b6b4-9b0d-52e8-9e09-a9e9cf4b5a4d'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='organization', description='organization'), description='An organization where David Thompson worked as a Graphic Designer.'), Entity(id=UUID('60b027fe-7bb4-535d-8a47-19f1a491591b'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='rhode island school of design', is_a=EntityType(id=UUID('b5866225-05ad-5cfc-908e-c22916c6a1c6'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='institution', description='institution'), description='Educational institution where David Thompson earned his B.F.A. in Graphic Design.'), Entity(id=UUID('7e3df89c-2691-580b-84dc-378cb1df3db6'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='adobe creative suite', is_a=EntityType(id=UUID('2d66edc2-1e14-55ab-8304-680b514a597a'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='software', description='software'), description='A suite of graphic design, video editing, and web development applications.'), Entity(id=UUID('2a0f9b58-c695-5bad-baa2-fd2da02da013'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='html', is_a=EntityType(id=UUID('c90c7d6b-3532-5dcf-91e1-4a0e1f179794'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='language', description='language'), description='A markup language for creating web pages and applications.'), Entity(id=UUID('9b062f3c-fe02-5427-9b44-b287a1cac367'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='css', is_a=EntityType(id=UUID('c90c7d6b-3532-5dcf-91e1-4a0e1f179794'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='language', description='language'), description='A stylesheet language used for describing the presentation of a document written in HTML.')])), TextSummary(id=UUID('8aedca6b-fa78-5987-a79b-3b0bebff8eb1'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='Data Scientist with 6 years of expertise in machine learning, focused on utilizing data for business improvement and product enhancement.', made_from=DocumentChunk(id=UUID('00692e43-9f02-54d0-b695-44bf47342d36'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, text='\\nCV 3: Relevant\\nName: Sarah Nguyen\\nContact Information:\\n\\nEmail: sarah.nguyen@example.com\\nPhone: (555) 345-6789\\nSummary:\\n\\nData Scientist specializing in machine learning with 6 years of experience. Passionate about leveraging data to drive business solutions and improve product performance.\\n\\nEducation:\\n\\nM.S. in Statistics, University of Washington (2014)\\nB.S. in Applied Mathematics, University of Texas at Austin (2012)\\nExperience:\\n\\nData Scientist, QuantumTech (2016 – Present)\\nDesigned and implemented machine learning algorithms for financial forecasting.\\nImproved model efficiency by 20% through algorithm optimization.\\nJunior Data Scientist, DataCore Solutions (2014 – 2016)\\nAssisted in developing predictive models for supply chain optimization.\\nConducted data cleaning and preprocessing on large datasets.\\nSkills:\\n\\nProgramming Languages: Python, R\\nMachine Learning Frameworks: PyTorch, Scikit-Learn\\nStatistical Analysis: SAS, SPSS\\nCloud Platforms: AWS, Azure\\n', word_count=110, chunk_index=0, cut_type='sentence_cut', is_part_of=TextDocument(id=UUID('e7d6246b-e414-5b9d-8daa-6d4434b7fa47'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='text_f8c7482e727f228001b0046ed68d656f', raw_data_location='/Users/vasilije/cognee/cognee/.data_storage/data/text_f8c7482e727f228001b0046ed68d656f.txt', metadata_id=UUID('e7d6246b-e414-5b9d-8daa-6d4434b7fa47'), mime_type='text/plain', type='text'), contains=[Entity(id=UUID('4d8dda57-2681-5264-a2bd-e2ddfe66a785'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='sarah nguyen', is_a=EntityType(id=UUID('d072ba0f-e1a9-58bf-9974-e1802adc8134'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='person', description='person'), description='Data Scientist specializing in machine learning with 6 years of experience.'), Entity(id=UUID('ae74a35b-d5f1-5622-ade1-6703d5e069fb'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='university of washington', is_a=EntityType(id=UUID('912b273c-683d-53ea-8ffe-aadef0b84237'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='educational institution', description='educational institution'), description='University located in Seattle, Washington.'), Entity(id=UUID('301b3cf8-5a5c-585e-80bd-f79901e4368c'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='university of texas at austin', is_a=EntityType(id=UUID('912b273c-683d-53ea-8ffe-aadef0b84237'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='educational institution', description='educational institution'), description='Public research university located in Austin, Texas.'), Entity(id=UUID('0d980f2a-09dd-581e-acc3-cc2d87c1bab4'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='quantumtech', is_a=EntityType(id=UUID('a6ed6bf1-fe31-5dfe-8ab4-484691fdf219'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='company', description='company'), description='Company where Sarah Nguyen works as a Data Scientist from 2016 to present.'), Entity(id=UUID('95ac0551-38fc-5187-a422-533aeb7e8db0'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='datacore solutions', is_a=EntityType(id=UUID('a6ed6bf1-fe31-5dfe-8ab4-484691fdf219'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='company', description='company'), description='Company where Sarah Nguyen worked as a Junior Data Scientist from 2014 to 2016.'), Entity(id=UUID('3edcdf3f-25af-57a3-8878-8008bd7ea05a'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='aws', is_a=EntityType(id=UUID('d84d991a-dab3-5c36-8806-df076ccb731b'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='cloud platform', description='cloud platform'), description='Amazon Web Services, a cloud computing platform.'), Entity(id=UUID('8b431923-4aa2-5886-a661-b8de0f888a9b'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='azure', is_a=EntityType(id=UUID('d84d991a-dab3-5c36-8806-df076ccb731b'), updated_at=datetime.datetime(2024, 12, 24, 11, 54, 13, 481297, tzinfo=datetime.timezone.utc), topological_rank=0, name='cloud platform', description='cloud platform'), description='Microsoft Azure, a cloud computing service created by Microsoft.')]))]\n" - ] - } - ], - "execution_count": 11 + ] }, { "cell_type": "markdown", @@ -782,6 +713,7 @@ }, { "cell_type": "code", + "execution_count": null, "id": "080389e5", "metadata": { "ExecuteTime": { @@ -789,6 +721,7 @@ "start_time": "2024-12-29T16:55:46.922951Z" } }, + "outputs": [], "source": [ "import os\n", "from cognee.shared.utils import render_graph\n", @@ -801,42 +734,23 @@ "\n", "graph_url = await render_graph(graph_engine.graph)\n", "print(graph_url)" - ], - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Graph is visualized at: https://hub.graphistry.com/graph/graph.html?dataset=cc21b1d2d6074323aa37af53e693b1a4&type=arrow&viztoken=db05565e-79e9-4fe3-99b2-b7a2e6d48eff&usertag=5f822e63-pygraphistry-0.33.9&splashAfter=1735491366&info=true\n", - "https://hub.graphistry.com/graph/graph.html?dataset=cc21b1d2d6074323aa37af53e693b1a4&type=arrow&viztoken=db05565e-79e9-4fe3-99b2-b7a2e6d48eff&usertag=5f822e63-pygraphistry-0.33.9&splashAfter=1735491366&info=true\n" - ] - } - ], - "execution_count": 12 + ] }, { + "cell_type": "code", + "execution_count": null, + "id": "8f69caa0e353a889", "metadata": { "ExecuteTime": { "end_time": "2024-12-29T16:56:06.571404Z", "start_time": "2024-12-29T16:56:06.569280Z" } }, - "cell_type": "code", + "outputs": [], "source": [ "graph_engine = await get_graph_engine()\n", "print(graph_url)" - ], - "id": "8f69caa0e353a889", - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "https://hub.graphistry.com/graph/graph.html?dataset=cc21b1d2d6074323aa37af53e693b1a4&type=arrow&viztoken=db05565e-79e9-4fe3-99b2-b7a2e6d48eff&usertag=5f822e63-pygraphistry-0.33.9&splashAfter=1735491366&info=true\n" - ] - } - ], - "execution_count": 13 + ] }, { "cell_type": "markdown", @@ -848,6 +762,7 @@ }, { "cell_type": "code", + "execution_count": null, "id": "e5e7dfc8", "metadata": { "ExecuteTime": { @@ -855,6 +770,7 @@ "start_time": "2024-12-24T13:44:16.047897Z" } }, + "outputs": [], "source": [ "async def search(\n", " vector_engine,\n", @@ -883,26 +799,7 @@ "results = await search(vector_engine, \"entity_name\", \"sarah.nguyen@example.com\")\n", "for result in results:\n", " print(result)" - ], - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'id': '4d8dda57-2681-5264-a2bd-e2ddfe66a785', 'payload': {'id': '4d8dda57-2681-5264-a2bd-e2ddfe66a785', 'updated_at': datetime.datetime(2024, 12, 24, 11, 54, 13, 481297), 'topological_rank': 0, 'text': 'sarah nguyen'}, 'score': 0.5708460211753845}\n", - "{'id': '198e2ab8-75e9-5931-97ab-da9a5a8e188c', 'payload': {'id': '198e2ab8-75e9-5931-97ab-da9a5a8e188c', 'updated_at': datetime.datetime(2024, 12, 24, 11, 54, 13, 481297), 'topological_rank': 0, 'text': 'san francisco, ca'}, 'score': 1.349550724029541}\n", - "{'id': '435dbd37-ab20-503c-9e99-ab8b8a3484e5', 'payload': {'id': '435dbd37-ab20-503c-9e99-ab8b8a3484e5', 'updated_at': datetime.datetime(2024, 12, 24, 11, 54, 13, 481297), 'topological_rank': 0, 'text': 'senior data scientist'}, 'score': 1.3934645652770996}\n", - "{'id': '36a5e3c8-c5f5-5ab5-8d59-ea69d8b36932', 'payload': {'id': '36a5e3c8-c5f5-5ab5-8d59-ea69d8b36932', 'updated_at': datetime.datetime(2024, 12, 24, 11, 54, 13, 481297), 'topological_rank': 0, 'text': 'jessica miller'}, 'score': 1.4042469263076782}\n", - "{'id': '73ae630f-7b09-5dce-8c18-45d0a57b30f9', 'payload': {'id': '73ae630f-7b09-5dce-8c18-45d0a57b30f9', 'updated_at': datetime.datetime(2024, 12, 24, 11, 54, 13, 481297), 'topological_rank': 0, 'text': 'michael rodriguez'}, 'score': 1.4521806240081787}\n", - "{'id': '29e771c8-4c3f-52de-9511-6b705878e130', 'payload': {'id': '29e771c8-4c3f-52de-9511-6b705878e130', 'updated_at': datetime.datetime(2024, 12, 24, 11, 54, 13, 481297), 'topological_rank': 0, 'text': 'dr. emily carter'}, 'score': 1.471205472946167}\n", - "{'id': 'ce8b394a-b30e-52fc-b80a-6352edc60e5b', 'payload': {'id': 'ce8b394a-b30e-52fc-b80a-6352edc60e5b', 'updated_at': datetime.datetime(2024, 12, 24, 11, 54, 13, 481297), 'topological_rank': 0, 'text': 'stanford university'}, 'score': 1.4871069192886353}\n", - "{'id': '9780afb1-dccc-53eb-9a30-c0d4ce033711', 'payload': {'id': '9780afb1-dccc-53eb-9a30-c0d4ce033711', 'updated_at': datetime.datetime(2024, 12, 24, 11, 54, 13, 481297), 'topological_rank': 0, 'text': 'innovateai labs'}, 'score': 1.498255968093872}\n", - "{'id': '301b3cf8-5a5c-585e-80bd-f79901e4368c', 'payload': {'id': '301b3cf8-5a5c-585e-80bd-f79901e4368c', 'updated_at': datetime.datetime(2024, 12, 24, 11, 54, 13, 481297), 'topological_rank': 0, 'text': 'university of texas at austin'}, 'score': 1.5053001642227173}\n", - "{'id': '2c02c93c-9cd1-56b8-9cc0-55ff0b290e57', 'payload': {'id': '2c02c93c-9cd1-56b8-9cc0-55ff0b290e57', 'updated_at': datetime.datetime(2024, 12, 24, 11, 54, 13, 481297), 'topological_rank': 0, 'text': 'university of california, berkeley'}, 'score': 1.5213639736175537}\n" - ] - } - ], - "execution_count": 23 + ] }, { "cell_type": "markdown", @@ -981,455 +878,351 @@ ] }, { + "cell_type": "markdown", + "id": "c9ffa271", + "metadata": {}, + "source": [ + "### Now let's add evals" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afae18ac6a794925", "metadata": { "ExecuteTime": { "end_time": "2024-12-24T13:46:09.644509Z", "start_time": "2024-12-24T13:46:04.538592Z" } }, - "cell_type": "code", + "outputs": [], "source": [ - "!pip install wget\n", - "!pip install deepeval\n", - "!pip install ujson" - ], - "id": "afae18ac6a794925", - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: wget in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (3.2)\r\n", - "\r\n", - "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\r\n", - "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\r\n", - "Requirement already satisfied: deepeval in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (2.0.9)\r\n", - "Requirement already satisfied: requests in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (2.32.3)\r\n", - "Requirement already satisfied: tqdm in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (4.67.1)\r\n", - "Requirement already satisfied: pytest in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (7.4.4)\r\n", - "Requirement already satisfied: tabulate in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (0.9.0)\r\n", - "Requirement already satisfied: typer in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (0.15.1)\r\n", - "Requirement already satisfied: rich in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (13.9.4)\r\n", - "Requirement already satisfied: protobuf in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (4.25.5)\r\n", - "Requirement already satisfied: pydantic in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (2.8.2)\r\n", - "Requirement already satisfied: sentry-sdk in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (2.19.2)\r\n", - "Requirement already satisfied: pytest-repeat in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (0.9.3)\r\n", - "Requirement already satisfied: pytest-xdist in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (3.6.1)\r\n", - "Requirement already satisfied: portalocker in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (3.0.0)\r\n", - "Requirement already satisfied: langchain in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (0.3.13)\r\n", - "Requirement already satisfied: langchain-core in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (0.3.28)\r\n", - "Requirement already satisfied: langchain_openai in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (0.2.14)\r\n", - "Requirement already satisfied: langchain-community in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (0.3.13)\r\n", - "Requirement already satisfied: docx2txt~=0.8 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (0.8)\r\n", - "Requirement already satisfied: importlib-metadata>=6.0.2 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (8.4.0)\r\n", - "Requirement already satisfied: tenacity<=9.0.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (8.5.0)\r\n", - "Requirement already satisfied: opentelemetry-api<2.0.0,>=1.24.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (1.27.0)\r\n", - "Requirement already satisfied: opentelemetry-sdk<2.0.0,>=1.24.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (1.27.0)\r\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (1.27.0)\r\n", - "Requirement already satisfied: grpcio==1.60.1 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (1.60.1)\r\n", - "Requirement already satisfied: nest-asyncio in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deepeval) (1.6.0)\r\n", - "Requirement already satisfied: zipp>=0.5 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from importlib-metadata>=6.0.2->deepeval) (3.21.0)\r\n", - "Requirement already satisfied: deprecated>=1.2.6 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from opentelemetry-api<2.0.0,>=1.24.0->deepeval) (1.2.15)\r\n", - "Requirement already satisfied: googleapis-common-protos~=1.52 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval) (1.66.0)\r\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.27.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval) (1.27.0)\r\n", - "Requirement already satisfied: opentelemetry-proto==1.27.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from opentelemetry-exporter-otlp-proto-grpc<2.0.0,>=1.24.0->deepeval) (1.27.0)\r\n", - "Requirement already satisfied: opentelemetry-semantic-conventions==0.48b0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from opentelemetry-sdk<2.0.0,>=1.24.0->deepeval) (0.48b0)\r\n", - "Requirement already satisfied: typing-extensions>=3.7.4 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from opentelemetry-sdk<2.0.0,>=1.24.0->deepeval) (4.12.2)\r\n", - "Requirement already satisfied: PyYAML>=5.3 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain->deepeval) (6.0.2)\r\n", - "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain->deepeval) (2.0.35)\r\n", - "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain->deepeval) (3.10.10)\r\n", - "Requirement already satisfied: langchain-text-splitters<0.4.0,>=0.3.3 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain->deepeval) (0.3.4)\r\n", - "Requirement already satisfied: langsmith<0.3,>=0.1.17 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain->deepeval) (0.2.4)\r\n", - "Requirement already satisfied: numpy<2,>=1.22.4 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain->deepeval) (1.26.4)\r\n", - "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain-core->deepeval) (1.33)\r\n", - "Requirement already satisfied: packaging<25,>=23.2 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain-core->deepeval) (24.2)\r\n", - "Requirement already satisfied: annotated-types>=0.4.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from pydantic->deepeval) (0.7.0)\r\n", - "Requirement already satisfied: pydantic-core==2.20.1 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from pydantic->deepeval) (2.20.1)\r\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from requests->deepeval) (3.4.0)\r\n", - "Requirement already satisfied: idna<4,>=2.5 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from requests->deepeval) (3.10)\r\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from requests->deepeval) (2.2.3)\r\n", - "Requirement already satisfied: certifi>=2017.4.17 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from requests->deepeval) (2024.12.14)\r\n", - "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain-community->deepeval) (0.6.7)\r\n", - "Requirement already satisfied: httpx-sse<0.5.0,>=0.4.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain-community->deepeval) (0.4.0)\r\n", - "Requirement already satisfied: pydantic-settings<3.0.0,>=2.4.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain-community->deepeval) (2.7.0)\r\n", - "Requirement already satisfied: openai<2.0.0,>=1.58.1 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain_openai->deepeval) (1.58.1)\r\n", - "Requirement already satisfied: tiktoken<1,>=0.7 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langchain_openai->deepeval) (0.7.0)\r\n", - "Requirement already satisfied: iniconfig in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from pytest->deepeval) (2.0.0)\r\n", - "Requirement already satisfied: pluggy<2.0,>=0.12 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from pytest->deepeval) (1.5.0)\r\n", - "Requirement already satisfied: execnet>=2.1 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from pytest-xdist->deepeval) (2.1.1)\r\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from rich->deepeval) (3.0.0)\r\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from rich->deepeval) (2.18.0)\r\n", - "Requirement already satisfied: click>=8.0.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from typer->deepeval) (8.1.7)\r\n", - "Requirement already satisfied: shellingham>=1.3.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from typer->deepeval) (1.5.4)\r\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain->deepeval) (2.4.4)\r\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain->deepeval) (1.3.2)\r\n", - "Requirement already satisfied: attrs>=17.3.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain->deepeval) (24.3.0)\r\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain->deepeval) (1.5.0)\r\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain->deepeval) (6.1.0)\r\n", - "Requirement already satisfied: yarl<2.0,>=1.12.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain->deepeval) (1.18.3)\r\n", - "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community->deepeval) (3.23.2)\r\n", - "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community->deepeval) (0.9.0)\r\n", - "Requirement already satisfied: wrapt<2,>=1.10 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from deprecated>=1.2.6->opentelemetry-api<2.0.0,>=1.24.0->deepeval) (1.17.0)\r\n", - "Requirement already satisfied: jsonpointer>=1.9 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from jsonpatch<2.0,>=1.33->langchain-core->deepeval) (3.0.0)\r\n", - "Requirement already satisfied: httpx<1,>=0.23.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langsmith<0.3,>=0.1.17->langchain->deepeval) (0.27.0)\r\n", - "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langsmith<0.3,>=0.1.17->langchain->deepeval) (3.10.12)\r\n", - "Requirement already satisfied: requests-toolbelt<2.0.0,>=1.0.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from langsmith<0.3,>=0.1.17->langchain->deepeval) (1.0.0)\r\n", - "Requirement already satisfied: mdurl~=0.1 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich->deepeval) (0.1.2)\r\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from openai<2.0.0,>=1.58.1->langchain_openai->deepeval) (4.7.0)\r\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from openai<2.0.0,>=1.58.1->langchain_openai->deepeval) (1.9.0)\r\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from openai<2.0.0,>=1.58.1->langchain_openai->deepeval) (0.5.0)\r\n", - "Requirement already satisfied: sniffio in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from openai<2.0.0,>=1.58.1->langchain_openai->deepeval) (1.3.1)\r\n", - "Requirement already satisfied: python-dotenv>=0.21.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from pydantic-settings<3.0.0,>=2.4.0->langchain-community->deepeval) (1.0.1)\r\n", - "Requirement already satisfied: regex>=2022.1.18 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from tiktoken<1,>=0.7->langchain_openai->deepeval) (2024.11.6)\r\n", - "Requirement already satisfied: httpcore==1.* in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from httpx<1,>=0.23.0->langsmith<0.3,>=0.1.17->langchain->deepeval) (1.0.7)\r\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<0.3,>=0.1.17->langchain->deepeval) (0.14.0)\r\n", - "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain-community->deepeval) (1.0.0)\r\n", - "Requirement already satisfied: propcache>=0.2.0 in /Users/vasilije/cognee/.venv/lib/python3.11/site-packages (from yarl<2.0,>=1.12.0->aiohttp<4.0.0,>=3.8.3->langchain->deepeval) (0.2.1)\r\n", - "\r\n", - "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\r\n", - "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\r\n", - "Collecting ujson\r\n", - " Obtaining dependency information for ujson from https://files.pythonhosted.org/packages/8d/9f/4731ef0671a0653e9f5ba18db7c4596d8ecbf80c7922dd5fe4150f1aea76/ujson-5.10.0-cp311-cp311-macosx_11_0_arm64.whl.metadata\r\n", - " Downloading ujson-5.10.0-cp311-cp311-macosx_11_0_arm64.whl.metadata (9.3 kB)\r\n", - "Downloading ujson-5.10.0-cp311-cp311-macosx_11_0_arm64.whl (51 kB)\r\n", - "\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m51.8/51.8 kB\u001B[0m \u001B[31m1.7 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n", - "\u001B[?25hInstalling collected packages: ujson\r\n", - "Successfully installed ujson-5.10.0\r\n", - "\r\n", - "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m23.2.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m24.3.1\u001B[0m\r\n", - "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\r\n" - ] - } - ], - "execution_count": 29 + "!pip install \"cognee[deepeval]\"" + ] }, { + "cell_type": "code", + "execution_count": null, + "id": "5f36b67668fdb646", "metadata": { "ExecuteTime": { "end_time": "2024-12-24T15:29:11.123483Z", "start_time": "2024-12-24T15:29:11.120888Z" } }, - "cell_type": "code", - "source": [ - "# from evals.eval_on_hotpot import eval_on_hotpotQA\n", - "# from evals.eval_on_hotpot import answer_with_cognee\n", - "# from evals.eval_on_hotpot import answer_without_cognee\n", - "# from evals.eval_on_hotpot import eval_answers\n", - "# from cognee.base_config import get_base_config\n", - "# from pathlib import Path\n", - "# from tqdm import tqdm\n", - "# import wget\n", - "# import json\n", - "# import statistics" - ], - "id": "5f36b67668fdb646", "outputs": [], - "execution_count": 2 + "source": [ + "from evals.eval_on_hotpot import deepeval_answers, answer_qa_instance\n", + "from evals.qa_dataset_utils import load_qa_dataset\n", + "from evals.qa_metrics_utils import get_metrics\n", + "from evals.qa_context_provider_utils import qa_context_providers\n", + "from pathlib import Path\n", + "from tqdm import tqdm\n", + "import statistics\n", + "import random" + ] }, { - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-24T15:57:30.764970Z", - "start_time": "2024-12-24T15:57:07.861187Z" - } - }, "cell_type": "code", - "source": [ - "# answer_provider = answer_without_cognee # For native LLM answers use answer_without_cognee\n", - "# num_samples = 10 # With cognee, it takes ~1m10s per sample\n", - "# \n", - "# base_config = get_base_config()\n", - "# data_root_dir = base_config.data_root_directory\n", - "# \n", - "# if not Path(data_root_dir).exists():\n", - "# Path(data_root_dir).mkdir()\n", - "# \n", - "# filepath = data_root_dir / Path(\"hotpot_dev_fullwiki_v1.json\")\n", - "# if not filepath.exists():\n", - "# url = 'http://curtis.ml.cmu.edu/datasets/hotpot/hotpot_dev_fullwiki_v1.json'\n", - "# wget.download(url, out=data_root_dir)\n", - "# \n", - "# with open(filepath, \"r\") as file:\n", - "# dataset = json.load(file)\n", - "# instances = dataset if not num_samples else dataset[:num_samples]\n", - "# answers = []\n", - "# for instance in tqdm(instances, desc=\"Getting answers\"):\n", - "# answer = await answer_provider(instance)\n", - "# answers.append(answer)" - ], - "id": "d5af4b516c6621a3", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Getting answers: 100%|██████████| 10/10 [00:13<00:00, 1.31s/it]\n" - ] - } - ], - "execution_count": 9 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-24T15:57:30.787382Z", - "start_time": "2024-12-24T15:57:30.785259Z" - } - }, - "cell_type": "code", - "source": [ - "# from evals.deepeval_metrics import f1_score_metric\n", - "# from evals.deepeval_metrics import em_score_metric" - ], - "id": "2bf69048a272158c", + "execution_count": null, + "id": "de91ee0a", + "metadata": {}, "outputs": [], - "execution_count": 10 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-24T15:57:30.828509Z", - "start_time": "2024-12-24T15:57:30.795197Z" - } - }, - "cell_type": "code", "source": [ - "# f1_metric = f1_score_metric()\n", - "# eval_results = await eval_answers(instances, answers, f1_metric)\n", - "# avg_f1_score = statistics.mean([result.metrics_data[0].score for result in eval_results.test_results])\n", - "# print(\"F1 score: \", avg_f1_score)" - ], - "id": "72ba5f89cccbee6b", - "outputs": [ - { - "data": { - "text/plain": [ - "✨ You're running DeepEval's latest \u001B[38;2;106;0;255mOfficial hotpot F1 score Metric\u001B[0m! \u001B[1;38;2;55;65;81m(\u001B[0m\u001B[38;2;55;65;81musing \u001B[0m\u001B[3;38;2;55;65;81mNone\u001B[0m\u001B[38;2;55;65;81m, \u001B[0m\u001B[38;2;55;65;81mstrict\u001B[0m\u001B[38;2;55;65;81m=\u001B[0m\u001B[3;38;2;55;65;81mFalse\u001B[0m\u001B[38;2;55;65;81m, \u001B[0m\u001B[38;2;55;65;81masync_mode\u001B[0m\u001B[38;2;55;65;81m=\u001B[0m\u001B[3;38;2;55;65;81mTrue\u001B[0m\u001B[1;38;2;55;65;81m)\u001B[0m\u001B[38;2;55;65;81m...\u001B[0m\n" - ], - "text/html": [ - "
✨ You're running DeepEval's latest Official hotpot F1 score Metric! (using None, strict=False, async_mode=True)...\n",
-       "
\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Event loop is already running. Applying nest_asyncio patch to allow async execution...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Evaluating 10 test case(s) in parallel: |██████████|100% (10/10) [Time Taken: 00:00, 407.84test case/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "======================================================================\n", - "\n", - "Metrics Summary\n", - "\n", - " - ❌ Official hotpot F1 score (score: 0.0, threshold: 0.5, strict: False, evaluation model: None, reason: None, error: None)\n", - "\n", - "For test case:\n", - "\n", - " - input: Were Scott Derrickson and Ed Wood of the same nationality?\n", - " - actual output: Scott Derrickson is described as an American filmmaker in the context. Ed Wood is referenced as a cult filmmaker in the American biographical film \"Ed Wood.\" Therefore, both Scott Derrickson and Ed Wood are of the same nationality: American.\n", - " - expected output: yes\n", - " - context: None\n", - " - retrieval context: None\n", - "\n", - "======================================================================\n", - "\n", - "Metrics Summary\n", - "\n", - " - ❌ Official hotpot F1 score (score: 0.04255319148936171, threshold: 0.5, strict: False, evaluation model: None, reason: None, error: None)\n", - "\n", - "For test case:\n", - "\n", - " - input: What government position was held by the woman who portrayed Corliss Archer in the film Kiss and Tell?\n", - " - actual output: The woman who portrayed Corliss Archer in the film \"Kiss and Tell\" was Shirley Temple. After her acting career, she held the position of U.S. Ambassador to Czechoslovakia from 1989 to 1992, and she was also a member of the U.S. delegation to the United Nations from 1969 to 1970.\n", - " - expected output: Chief of Protocol\n", - " - context: None\n", - " - retrieval context: None\n", - "\n", - "======================================================================\n", - "\n", - "Metrics Summary\n", - "\n", - " - ❌ Official hotpot F1 score (score: 0.05882352941176471, threshold: 0.5, strict: False, evaluation model: None, reason: None, error: None)\n", - "\n", - "For test case:\n", - "\n", - " - input: What science fantasy young adult series, told in first person, has a set of companion books narrating the stories of enslaved worlds and alien species?\n", - " - actual output: The book series that fits the description is \"Animorphs\". It is a science fantasy young adult series told in the first person with multiple narrators, and it deals with themes relevant to enslaved worlds and alien species.\n", - " - expected output: Animorphs\n", - " - context: None\n", - " - retrieval context: None\n", - "\n", - "======================================================================\n", - "\n", - "Metrics Summary\n", - "\n", - " - ❌ Official hotpot F1 score (score: 0.0, threshold: 0.5, strict: False, evaluation model: None, reason: None, error: None)\n", - "\n", - "For test case:\n", - "\n", - " - input: Are the Laleli Mosque and Esma Sultan Mansion located in the same neighborhood?\n", - " - actual output: The Laleli Mosque is located in Laleli, Fatih, Istanbul. The Esma Sultan Mansion is located in the Ortaköy neighborhood of Istanbul. Therefore, they are not located in the same neighborhood.\n", - " - expected output: no\n", - " - context: None\n", - " - retrieval context: None\n", - "\n", - "======================================================================\n", - "\n", - "Metrics Summary\n", - "\n", - " - ❌ Official hotpot F1 score (score: 0.22222222222222218, threshold: 0.5, strict: False, evaluation model: None, reason: None, error: None)\n", - "\n", - "For test case:\n", - "\n", - " - input: The director of the romantic comedy \"Big Stone Gap\" is based in what New York city?\n", - " - actual output: The director of the romantic comedy \"Big Stone Gap\" is Adriana Trigiani, and there is no specific New York city mentioned in the provided context.\n", - " - expected output: Greenwich Village, New York City\n", - " - context: None\n", - " - retrieval context: None\n", - "\n", - "======================================================================\n", - "\n", - "Metrics Summary\n", - "\n", - " - ❌ Official hotpot F1 score (score: 0.11764705882352941, threshold: 0.5, strict: False, evaluation model: None, reason: None, error: None)\n", - "\n", - "For test case:\n", - "\n", - " - input: 2014 S/S is the debut album of a South Korean boy group that was formed by who?\n", - " - actual output: The South Korean boy group that debuted with the album \"2014 S/S\" was formed by FNC Entertainment.\n", - " - expected output: YG Entertainment\n", - " - context: None\n", - " - retrieval context: None\n", - "\n", - "======================================================================\n", - "\n", - "Metrics Summary\n", - "\n", - " - ❌ Official hotpot F1 score (score: 0.18181818181818182, threshold: 0.5, strict: False, evaluation model: None, reason: None, error: None)\n", - "\n", - "For test case:\n", - "\n", - " - input: Who was known by his stage name Aladin and helped organizations improve their performance as a consultant?\n", - " - actual output: Eenasul Fateh, known by his stage name Aladin, is a Bangladeshi-British cultural practitioner, magician, live artist, and former international management consultant.\n", - " - expected output: Eenasul Fateh\n", - " - context: None\n", - " - retrieval context: None\n", - "\n", - "======================================================================\n", - "\n", - "Metrics Summary\n", - "\n", - " - ❌ Official hotpot F1 score (score: 0.0, threshold: 0.5, strict: False, evaluation model: None, reason: None, error: None)\n", - "\n", - "For test case:\n", - "\n", - " - input: The arena where the Lewiston Maineiacs played their home games can seat how many people?\n", - " - actual output: The Lewiston Maineiacs played their home games at the Androscoggin Bank Colisée. The capacity of the arena is not directly stated in the context, but it is implied from the information provided.\n", - " - expected output: 3,677 seated\n", - " - context: None\n", - " - retrieval context: None\n", - "\n", - "======================================================================\n", - "\n", - "Metrics Summary\n", - "\n", - " - ❌ Official hotpot F1 score (score: 0.12903225806451613, threshold: 0.5, strict: False, evaluation model: None, reason: None, error: None)\n", - "\n", - "For test case:\n", - "\n", - " - input: Who is older, Annie Morton or Terry Richardson?\n", - " - actual output: Annie Morton was born on October 8, 1970, while Terry Richardson's birthdate is not provided in the context. Therefore, based on the available information, Annie Morton is older than Terry Richardson.\n", - " - expected output: Terry Richardson\n", - " - context: None\n", - " - retrieval context: None\n", - "\n", - "======================================================================\n", - "\n", - "Metrics Summary\n", - "\n", - " - ❌ Official hotpot F1 score (score: 0.0, threshold: 0.5, strict: False, evaluation model: None, reason: None, error: None)\n", - "\n", - "For test case:\n", - "\n", - " - input: Are Local H and For Against both from the United States?\n", - " - actual output: Yes, Local H is from the United States, specifically mentioned as a band from the Chicago suburbs, and For Against is also a band mentioned as being from the United States, although specific details about For Against's origin are not provided in the context.\n", - " - expected output: yes\n", - " - context: None\n", - " - retrieval context: None\n", - "\n", - "======================================================================\n", - "\n", - "Overall Metric Pass Rates\n", - "\n", - "Official hotpot F1 score: 0.00% pass rate\n", - "\n", - "======================================================================\n", - "\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - }, - { - "data": { - "text/plain": [ - "\u001B[38;2;5;245;141m✓\u001B[0m Tests finished 🎉! Run \u001B[32m'deepeval login'\u001B[0m to save and analyze evaluation results on Confident AI. \n", - "‼️ Friendly reminder 😇: You can also run evaluations with ALL of deepeval's metrics directly on Confident AI \n", - "instead.\n" - ], - "text/html": [ - "
✓ Tests finished 🎉! Run 'deepeval login' to save and analyze evaluation results on Confident AI. \n",
-       "‼️  Friendly reminder 😇: You can also run evaluations with ALL of deepeval's metrics directly on Confident AI \n",
-       "instead.\n",
-       "
\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "F1 score: 0.0752096441829576\n" - ] - } - ], - "execution_count": 11 + "num_samples = 10 # With cognee, it takes ~1m10s per sample\n", + "dataset_name_or_filename = \"hotpotqa\"\n", + "dataset = load_qa_dataset(dataset_name_or_filename)" + ] }, { - "metadata": { - "ExecuteTime": { - "end_time": "2025-01-05T19:23:30.332977Z", - "start_time": "2025-01-05T19:23:30.331538Z" - } - }, "cell_type": "code", - "source": "", - "id": "783985c35d1126de", + "execution_count": null, + "id": "04bbea26", + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "context_provider_name = \"cognee\"\n", + "context_provider = qa_context_providers[context_provider_name]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1194d32c", + "metadata": {}, + "outputs": [], + "source": [ + "random.seed(42)\n", + "instances = dataset if not num_samples else random.sample(dataset, num_samples)\n", + "\n", + "out_path = \"out\" \n", + "if not Path(out_path).exists():\n", + " Path(out_path).mkdir()\n", + "contexts_filename = out_path / Path(\n", + " f\"contexts_{dataset_name_or_filename.split('.')[0]}_{context_provider_name}.json\"\n", + " )\n", + "\n", + "answers = []\n", + "for instance in tqdm(instances, desc=\"Getting answers\"):\n", + " answer = await answer_qa_instance(instance, context_provider, contexts_filename)\n", + " answers.append(answer)" + ] + }, + { + "cell_type": "markdown", + "id": "7e8e491a", + "metadata": {}, + "source": [ + "#### Define Metrics for Evaluation and Calculate Score\n", + "**Options**: \n", + "- **Correctness**: Is the actual output factually correct based on the expected output?\n", + "- **Comprehensiveness**: How much detail does the answer provide to cover all aspects and details of the question?\n", + "- **Diversity**: How varied and rich is the answer in providing different perspectives and insights on the question?\n", + "- **Empowerment**: How well does the answer help the reader understand and make informed judgements about the topic?\n", + "- **Directness**: How specifically and clearly does the answer address the question?\n", + "- **F1 Score**: the harmonic mean of the precision and recall, using word-level Exact Match\n", + "- **EM Score**: the rate at which the predicted strings exactly match their references, ignoring white spaces and capitalization." + ] + }, + { + "cell_type": "markdown", + "id": "a65c70dd", + "metadata": {}, + "source": [ + "##### Calculate `\"Correctness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ede84200", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Correctness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df790d87", + "metadata": {}, + "outputs": [], + "source": [ + "Correctness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Correctness)" + ] + }, + { + "cell_type": "markdown", + "id": "a3dad48f", + "metadata": {}, + "source": [ + "##### Calculating `\"Comprehensiveness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20f98ae2", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Comprehensiveness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b35110d3", + "metadata": {}, + "outputs": [], + "source": [ + "Comprehensiveness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Comprehensiveness)" + ] + }, + { + "cell_type": "markdown", + "id": "9020eaaa", + "metadata": {}, + "source": [ + "##### Calculating `\"Diversity\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5aa9c70", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Diversity\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa460a81", + "metadata": {}, + "outputs": [], + "source": [ + "Diversity = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Diversity)" + ] + }, + { + "cell_type": "markdown", + "id": "636b37f2", + "metadata": {}, + "source": [ + "##### Calculating`\"Empowerment\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a685df3", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Empowerment\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44125ae7", + "metadata": {}, + "outputs": [], + "source": [ + "Empowerment = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Empowerment)" + ] + }, + { + "cell_type": "markdown", + "id": "5837fa32", + "metadata": {}, + "source": [ + "##### Calculating `\"Directness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f6b32b6", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Directness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91c79165", + "metadata": {}, + "outputs": [], + "source": [ + "Directness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Directness)" + ] + }, + { + "cell_type": "markdown", + "id": "912293c4", + "metadata": {}, + "source": [ + "##### Calculating `\"F1\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a80d6f2e", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"F1\"]\n", + "eval_metrics = get_metrics(metric_name_list)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e644f1f", + "metadata": {}, + "outputs": [], + "source": [ + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12d00f5e", + "metadata": {}, + "outputs": [], + "source": [ + "F1_score = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(F1_score)" + ] + }, + { + "cell_type": "markdown", + "id": "68680dd7", + "metadata": {}, + "source": [ + "##### Calculating `\"EM\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dfe28005", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"EM\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01dffe4d", + "metadata": {}, + "outputs": [], + "source": [ + "EM = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(EM)" + ] }, { "cell_type": "markdown", @@ -1439,27 +1232,11 @@ "# Give us a star if you like it!\n", "https://github.com/topoteretes/cognee" ] - }, - { - "metadata": {}, - "cell_type": "code", - "outputs": [], - "execution_count": null, - "source": "", - "id": "d042efe5d38144fa" - }, - { - "metadata": {}, - "cell_type": "code", - "outputs": [], - "execution_count": null, - "source": "", - "id": "9436af97520e0ae" } ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "cognee-c83GrcRT-py3.11", "language": "python", "name": "python3" }, @@ -1473,7 +1250,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.8" + "version": "3.11.10" } }, "nbformat": 4, diff --git a/notebooks/cognee_hotpot_eval.ipynb b/notebooks/cognee_hotpot_eval.ipynb index 445a13c1d..b57ad4365 100644 --- a/notebooks/cognee_hotpot_eval.ipynb +++ b/notebooks/cognee_hotpot_eval.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Evaluation on the hotpotQA dataset" + "# Evaluation on the hotpotQA dataset" ] }, { @@ -13,23 +13,30 @@ "metadata": {}, "outputs": [], "source": [ - "from evals.eval_on_hotpot import eval_on_hotpotQA\n", - "from evals.eval_on_hotpot import answer_with_cognee\n", - "from evals.eval_on_hotpot import answer_without_cognee\n", - "from evals.eval_on_hotpot import eval_answers\n", - "from cognee.base_config import get_base_config\n", + "!pip install \"cognee[deepeval]\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from evals.eval_on_hotpot import deepeval_answers, answer_qa_instance\n", + "from evals.qa_dataset_utils import load_qa_dataset\n", + "from evals.qa_metrics_utils import get_metrics\n", + "from evals.qa_context_provider_utils import qa_context_providers\n", "from pathlib import Path\n", "from tqdm import tqdm\n", - "import wget\n", - "import json\n", - "import statistics" + "import statistics\n", + "import random" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Getting the answers for the first num_samples questions of the dataset" + "## Load Dataset" ] }, { @@ -38,27 +45,69 @@ "metadata": {}, "outputs": [], "source": [ - "answer_provider = answer_with_cognee # For native LLM answers use answer_without_cognee\n", "num_samples = 10 # With cognee, it takes ~1m10s per sample\n", + "dataset_name_or_filename = \"hotpotqa\"\n", + "dataset = load_qa_dataset(dataset_name_or_filename)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define Context Provider\n", + "**Options**: \n", + "- **no_rag**: raw context \n", + "- **cognee**: context with cognee \n", + "- **simple_rag**: context with simple rag \n", + "- **brute_force**: context with brute force triplet search" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Here, \"no_rag\" is used as context provider\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "context_provider_name = \"no_rag\"\n", + "context_provider = qa_context_providers[context_provider_name]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Generate Answers for QA Instances\n", + "1. **Random Sampling**: Selects a random subset of the dataset if `num_samples` is defined.\n", + "2. **Context Filename**: Defines the file path for storing contexts generated by the context provider.\n", + "3. **Answer Generation**: Iterates over the QA instances using `tqdm` for progress tracking and generates answers using the `answer_qa_instance` function asynchronously." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "random.seed(42)\n", + "instances = dataset if not num_samples else random.sample(dataset, num_samples)\n", "\n", - "base_config = get_base_config()\n", - "data_root_dir = base_config.data_root_directory\n", + "out_path = \"out\" \n", + "if not Path(out_path).exists():\n", + " Path(out_path).mkdir()\n", + "contexts_filename = out_path / Path(\n", + " f\"contexts_{dataset_name_or_filename.split('.')[0]}_{context_provider_name}.json\"\n", + " )\n", "\n", - "if not Path(data_root_dir).exists():\n", - " Path(data_root_dir).mkdir()\n", - "\n", - "filepath = data_root_dir / Path(\"hotpot_dev_fullwiki_v1.json\")\n", - "if not filepath.exists():\n", - " url = \"http://curtis.ml.cmu.edu/datasets/hotpot/hotpot_dev_fullwiki_v1.json\"\n", - " wget.download(url, out=data_root_dir)\n", - "\n", - "with open(filepath, \"r\") as file:\n", - " dataset = json.load(file)\n", - "\n", - "instances = dataset if not num_samples else dataset[:num_samples]\n", "answers = []\n", "for instance in tqdm(instances, desc=\"Getting answers\"):\n", - " answer = await answer_provider(instance)\n", + " answer = await answer_qa_instance(instance, context_provider, contexts_filename)\n", " answers.append(answer)" ] }, @@ -66,62 +115,22 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Calculating the official HotpotQA benchmark metrics: F1 score and EM" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from evals.deepeval_metrics import f1_score_metric\n", - "from evals.deepeval_metrics import em_score_metric" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "f1_metric = f1_score_metric()\n", - "eval_results = await eval_answers(instances, answers, f1_metric)\n", - "avg_f1_score = statistics.mean(\n", - " [result.metrics_data[0].score for result in eval_results.test_results]\n", - ")\n", - "print(\"F1 score: \", avg_f1_score)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "em_metric = em_score_metric()\n", - "eval_results = await eval_answers(instances, answers, em_metric)\n", - "avg_em_score = statistics.mean(\n", - " [result.metrics_data[0].score for result in eval_results.test_results]\n", - ")\n", - "print(\"EM score: \", avg_em_score)" + "#### Define Metrics for Evaluation and Calculate Score\n", + "**Options**: \n", + "- **Correctness**: Is the actual output factually correct based on the expected output?\n", + "- **Comprehensiveness**: How much detail does the answer provide to cover all aspects and details of the question?\n", + "- **Diversity**: How varied and rich is the answer in providing different perspectives and insights on the question?\n", + "- **Empowerment**: How well does the answer help the reader understand and make informed judgements about the topic?\n", + "- **Directness**: How specifically and clearly does the answer address the question?\n", + "- **F1 Score**: the harmonic mean of the precision and recall, using word-level Exact Match\n", + "- **EM Score**: the rate at which the predicted strings exactly match their references, ignoring white spaces and capitalization." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Calculating a custom metric called Correctness\n", - "##### Correctness is judged by an LLM" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "from evals.deepeval_metrics import correctness_metric" + "##### Calculating `\"Correctness\"`" ] }, { @@ -130,29 +139,28 @@ "metadata": {}, "outputs": [], "source": [ - "eval_results = await eval_answers(\n", - " instances, answers, correctness_metric\n", - ") # note that instantiation is not needed for correctness metric as it is already an instance\n", - "avg_correctness_score = statistics.mean(\n", + "metric_name_list = [\"Correctness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Correctness = statistics.mean(\n", " [result.metrics_data[0].score for result in eval_results.test_results]\n", ")\n", - "print(\"Correctness score: \", avg_correctness_score)" + "print(Correctness)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Using a metric from Deepeval" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "from deepeval.metrics import AnswerRelevancyMetric" + "##### Calculating `\"Comprehensiveness\"`" ] }, { @@ -161,21 +169,28 @@ "metadata": {}, "outputs": [], "source": [ - "relevancy_metric = AnswerRelevancyMetric()\n", - "eval_results = await eval_answers(\n", - " instances, answers, relevancy_metric\n", - ") # note that instantiation is not needed for correctness metric as it is already an instance\n", - "avg_relevancy_score = statistics.mean(\n", + "metric_name_list = [\"Comprehensiveness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Comprehensiveness = statistics.mean(\n", " [result.metrics_data[0].score for result in eval_results.test_results]\n", ")\n", - "print(\"Relevancy score: \", avg_relevancy_score)" + "print(Comprehensiveness)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Answering and eval in one step" + "##### Calculating `\"Diversity\"`" ] }, { @@ -184,12 +199,9 @@ "metadata": {}, "outputs": [], "source": [ - "answer_provider = answer_without_cognee\n", - "f1_metric = f1_score_metric()\n", - "f1_score = await eval_on_hotpotQA(\n", - " answer_provider, num_samples=10, eval_metric=f1_metric\n", - ") # takes ~1m10s per sample\n", - "print(\"F1 score: \", f1_score)" + "metric_name_list = [\"Diversity\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " ] }, { @@ -197,17 +209,958 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "Diversity = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Diversity)" + ] }, { "cell_type": "markdown", "metadata": {}, - "source": [] + "source": [ + "##### Calculating`\"Empowerment\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Empowerment\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Empowerment = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Empowerment)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"Directness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Directness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Directness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Directness)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"F1\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"F1\"]\n", + "eval_metrics = get_metrics(metric_name_list)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "F1_score = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(F1_score)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"EM\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"EM\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "EM = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(EM)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### \"cognee\" as context provider\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "context_provider_name = \"cognee\"\n", + "context_provider = qa_context_providers[context_provider_name]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Generate Answers for QA Instances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "random.seed(42)\n", + "instances = dataset if not num_samples else random.sample(dataset, num_samples)\n", + "\n", + "out_path = \"out\" \n", + "if not Path(out_path).exists():\n", + " Path(out_path).mkdir()\n", + "contexts_filename = out_path / Path(\n", + " f\"contexts_{dataset_name_or_filename.split('.')[0]}_{context_provider_name}.json\"\n", + " )\n", + "\n", + "answers = []\n", + "for instance in tqdm(instances, desc=\"Getting answers\"):\n", + " answer = await answer_qa_instance(instance, context_provider, contexts_filename)\n", + " answers.append(answer)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Define Metrics for Evaluation and Calculate Score" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculate `\"Correctness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Correctness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Correctness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Correctness)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"Comprehensiveness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Comprehensiveness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Comprehensiveness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Comprehensiveness)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"Diversity\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Diversity\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Diversity = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Diversity)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating`\"Empowerment\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Empowerment\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Empowerment = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Empowerment)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"Directness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Directness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Directness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Directness)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"F1\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"F1\"]\n", + "eval_metrics = get_metrics(metric_name_list)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "F1_score = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(F1_score)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"EM\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"EM\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "EM = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(EM)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### \"simple_rag\" as context provider\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "context_provider_name = \"simple_rag\"\n", + "context_provider = qa_context_providers[context_provider_name]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Generate Answers for QA Instances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "random.seed(42)\n", + "instances = dataset if not num_samples else random.sample(dataset, num_samples)\n", + "\n", + "out_path = \"out\" \n", + "if not Path(out_path).exists():\n", + " Path(out_path).mkdir()\n", + "contexts_filename = out_path / Path(\n", + " f\"contexts_{dataset_name_or_filename.split('.')[0]}_{context_provider_name}.json\"\n", + " )\n", + "\n", + "answers = []\n", + "for instance in tqdm(instances, desc=\"Getting answers\"):\n", + " answer = await answer_qa_instance(instance, context_provider, contexts_filename)\n", + " answers.append(answer)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Define Metrics for Evaluation and Calculate Score" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculate `\"Correctness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Correctness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Correctness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Correctness)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"Comprehensiveness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Comprehensiveness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Comprehensiveness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Comprehensiveness)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"Diversity\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Diversity\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Diversity = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Diversity)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating`\"Empowerment\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Empowerment\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Empowerment = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Empowerment)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"Directness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Directness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Directness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Directness)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"F1\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"F1\"]\n", + "eval_metrics = get_metrics(metric_name_list)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "F1_score = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(F1_score)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"EM\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"EM\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "EM = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(EM)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### \"brute_force\" as context provider\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "context_provider_name = \"brute_force\"\n", + "context_provider = qa_context_providers[context_provider_name]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Generate Answers for QA Instances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "random.seed(42)\n", + "instances = dataset if not num_samples else random.sample(dataset, num_samples)\n", + "\n", + "out_path = \"out\" \n", + "if not Path(out_path).exists():\n", + " Path(out_path).mkdir()\n", + "contexts_filename = out_path / Path(\n", + " f\"contexts_{dataset_name_or_filename.split('.')[0]}_{context_provider_name}.json\"\n", + " )\n", + "\n", + "answers = []\n", + "for instance in tqdm(instances, desc=\"Getting answers\"):\n", + " answer = await answer_qa_instance(instance, context_provider, contexts_filename)\n", + " answers.append(answer)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Define Metrics for Evaluation and Calculate Score" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculate `\"Correctness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Correctness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Correctness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Correctness)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"Comprehensiveness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Comprehensiveness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Comprehensiveness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Comprehensiveness)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"Diversity\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Diversity\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Diversity = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Diversity)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating`\"Empowerment\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Empowerment\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Empowerment = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Empowerment)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"Directness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Directness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "Directness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Directness)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"F1\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"F1\"]\n", + "eval_metrics = get_metrics(metric_name_list)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "F1_score = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(F1_score)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##### Calculating `\"EM\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"EM\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "EM = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(EM)" + ] } ], "metadata": { "kernelspec": { - "display_name": "myenv", + "display_name": "cognee-c83GrcRT-py3.11", "language": "python", "name": "python3" }, @@ -221,7 +1174,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.20" + "version": "3.11.10" } }, "nbformat": 4, diff --git a/notebooks/hr_demo.ipynb b/notebooks/hr_demo.ipynb index c6c3c3ed1..63df77775 100644 --- a/notebooks/hr_demo.ipynb +++ b/notebooks/hr_demo.ipynb @@ -618,76 +618,339 @@ "cell_type": "markdown", "id": "e519e30c0423c2a", "metadata": {}, - "source": "## Let's add evals" - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "b22ae3d868fa5606", - "metadata": { - "ExecuteTime": { - "end_time": "2024-12-19T18:01:11.387716Z", - "start_time": "2024-12-19T18:01:11.278042Z" - } - }, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'deepeval'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[3], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mevals\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01meval_on_hotpot\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m eval_on_hotpotQA\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mevals\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01meval_on_hotpot\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m answer_with_cognee\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mevals\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01meval_on_hotpot\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m answer_without_cognee\n", - "File \u001b[0;32m~/cognee/evals/eval_on_hotpot.py:7\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mstatistics\u001b[39;00m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpathlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Path\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mdeepeval\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetrics\u001b[39;00m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mwget\u001b[39;00m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mdeepeval\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdataset\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m EvaluationDataset\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'deepeval'" - ] - } - ], "source": [ - "from evals.eval_on_hotpot import eval_on_hotpotQA\n", - "from evals.eval_on_hotpot import answer_with_cognee\n", - "from evals.eval_on_hotpot import answer_without_cognee\n", - "from evals.eval_on_hotpot import eval_answers\n", - "from cognee.base_config import get_base_config\n", - "from pathlib import Path\n", - "from tqdm import tqdm\n", - "import wget\n", - "import json\n", - "import statistics" + "## Let's add evals" ] }, { "cell_type": "code", "execution_count": null, - "id": "728355d390e3a01b", + "id": "3845443e", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install \"cognee[deepeval]\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a2c3c70", + "metadata": {}, + "outputs": [], + "source": [ + "from evals.eval_on_hotpot import deepeval_answers, answer_qa_instance\n", + "from evals.qa_dataset_utils import load_qa_dataset\n", + "from evals.qa_metrics_utils import get_metrics\n", + "from evals.qa_context_provider_utils import qa_context_providers\n", + "from pathlib import Path\n", + "from tqdm import tqdm\n", + "import statistics\n", + "import random" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53a609d8", "metadata": {}, "outputs": [], "source": [ - "answer_provider = answer_with_cognee # For native LLM answers use answer_without_cognee\n", "num_samples = 10 # With cognee, it takes ~1m10s per sample\n", + "dataset_name_or_filename = \"hotpotqa\"\n", + "dataset = load_qa_dataset(dataset_name_or_filename)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7351ab8f", + "metadata": {}, + "outputs": [], + "source": [ + "context_provider_name = \"cognee\"\n", + "context_provider = qa_context_providers[context_provider_name]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9346115b", + "metadata": {}, + "outputs": [], + "source": [ + "random.seed(42)\n", + "instances = dataset if not num_samples else random.sample(dataset, num_samples)\n", "\n", - "base_config = get_base_config()\n", - "data_root_dir = base_config.data_root_directory\n", + "out_path = \"out\" \n", + "if not Path(out_path).exists():\n", + " Path(out_path).mkdir()\n", + "contexts_filename = out_path / Path(\n", + " f\"contexts_{dataset_name_or_filename.split('.')[0]}_{context_provider_name}.json\"\n", + " )\n", "\n", - "if not Path(data_root_dir).exists():\n", - " Path(data_root_dir).mkdir()\n", - "\n", - "filepath = data_root_dir / Path(\"hotpot_dev_fullwiki_v1.json\")\n", - "if not filepath.exists():\n", - " url = \"http://curtis.ml.cmu.edu/datasets/hotpot/hotpot_dev_fullwiki_v1.json\"\n", - " wget.download(url, out=data_root_dir)\n", - "\n", - "with open(filepath, \"r\") as file:\n", - " dataset = json.load(file)\n", - "\n", - "instances = dataset if not num_samples else dataset[:num_samples]\n", "answers = []\n", "for instance in tqdm(instances, desc=\"Getting answers\"):\n", - " answer = answer_provider(instance)\n", + " answer = await answer_qa_instance(instance, context_provider, contexts_filename)\n", " answers.append(answer)" ] }, + { + "cell_type": "markdown", + "id": "1e7d872d", + "metadata": {}, + "source": [ + "#### Define Metrics for Evaluation and Calculate Score\n", + "**Options**: \n", + "- **Correctness**: Is the actual output factually correct based on the expected output?\n", + "- **Comprehensiveness**: How much detail does the answer provide to cover all aspects and details of the question?\n", + "- **Diversity**: How varied and rich is the answer in providing different perspectives and insights on the question?\n", + "- **Empowerment**: How well does the answer help the reader understand and make informed judgements about the topic?\n", + "- **Directness**: How specifically and clearly does the answer address the question?\n", + "- **F1 Score**: the harmonic mean of the precision and recall, using word-level Exact Match\n", + "- **EM Score**: the rate at which the predicted strings exactly match their references, ignoring white spaces and capitalization." + ] + }, + { + "cell_type": "markdown", + "id": "c81e2b46", + "metadata": {}, + "source": [ + "##### Calculate `\"Correctness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ae728344", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Correctness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "764aac6d", + "metadata": {}, + "outputs": [], + "source": [ + "Correctness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Correctness)" + ] + }, + { + "cell_type": "markdown", + "id": "6d3bbdc5", + "metadata": {}, + "source": [ + "##### Calculating `\"Comprehensiveness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9793ef78", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Comprehensiveness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9add448a", + "metadata": {}, + "outputs": [], + "source": [ + "Comprehensiveness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Comprehensiveness)" + ] + }, + { + "cell_type": "markdown", + "id": "bce2fa25", + "metadata": {}, + "source": [ + "##### Calculating `\"Diversity\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f60a179e", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Diversity\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7ccbd0ab", + "metadata": {}, + "outputs": [], + "source": [ + "Diversity = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Diversity)" + ] + }, + { + "cell_type": "markdown", + "id": "191cab63", + "metadata": {}, + "source": [ + "##### Calculating`\"Empowerment\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "66bec0bf", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Empowerment\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1b043a8f", + "metadata": {}, + "outputs": [], + "source": [ + "Empowerment = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Empowerment)" + ] + }, + { + "cell_type": "markdown", + "id": "2cac3be9", + "metadata": {}, + "source": [ + "##### Calculating `\"Directness\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "adaa17c0", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"Directness\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a8f97c9", + "metadata": {}, + "outputs": [], + "source": [ + "Directness = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(Directness)" + ] + }, + { + "cell_type": "markdown", + "id": "1ad6feb8", + "metadata": {}, + "source": [ + "##### Calculating `\"F1\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bdc48259", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"F1\"]\n", + "eval_metrics = get_metrics(metric_name_list)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c43c17c8", + "metadata": {}, + "outputs": [], + "source": [ + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8bfcc46d", + "metadata": {}, + "outputs": [], + "source": [ + "F1_score = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(F1_score)" + ] + }, + { + "cell_type": "markdown", + "id": "2583f948", + "metadata": {}, + "source": [ + "##### Calculating `\"EM\"`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90a8f630", + "metadata": {}, + "outputs": [], + "source": [ + "metric_name_list = [\"EM\"]\n", + "eval_metrics = get_metrics(metric_name_list)\n", + "eval_results = await deepeval_answers(instances, answers, eval_metrics[\"deepeval_metrics\"]) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d1b1ea1", + "metadata": {}, + "outputs": [], + "source": [ + "EM = statistics.mean(\n", + " [result.metrics_data[0].score for result in eval_results.test_results]\n", + ")\n", + "print(EM)" + ] + }, { "cell_type": "markdown", "id": "288ab570", @@ -700,7 +963,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "cognee-c83GrcRT-py3.11", "language": "python", "name": "python3" }, @@ -714,7 +977,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.8" + "version": "3.11.10" } }, "nbformat": 4,