diff --git a/tests/evals/data/LongMemEval_Snippetization.ipynb b/tests/evals/data/LongMemEval_Snippetization.ipynb index 87e9aeaf..a7a7937f 100644 --- a/tests/evals/data/LongMemEval_Snippetization.ipynb +++ b/tests/evals/data/LongMemEval_Snippetization.ipynb @@ -1,1174 +1,510 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "i9i9uUZ3pWQE", - "outputId": "84404bb8-5841-4f2f-dd87-7f909c6e95aa" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting gdown\n", - " Using cached gdown-5.2.0-py3-none-any.whl.metadata (5.8 kB)\n", - "Requirement already satisfied: pandas in ./.venv/lib/python3.13/site-packages (2.2.3)\n", - "Collecting beautifulsoup4 (from gdown)\n", - " Using cached beautifulsoup4-4.12.3-py3-none-any.whl.metadata (3.8 kB)\n", - "Collecting filelock (from gdown)\n", - " Using cached filelock-3.16.1-py3-none-any.whl.metadata (2.9 kB)\n", - "Collecting requests[socks] (from gdown)\n", - " Using cached requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)\n", - "Collecting tqdm (from gdown)\n", - " Using cached tqdm-4.67.1-py3-none-any.whl.metadata (57 kB)\n", - "Requirement already satisfied: numpy>=1.26.0 in ./.venv/lib/python3.13/site-packages (from pandas) (2.2.1)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in ./.venv/lib/python3.13/site-packages (from pandas) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in ./.venv/lib/python3.13/site-packages (from pandas) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in ./.venv/lib/python3.13/site-packages (from pandas) (2024.2)\n", - "Requirement already satisfied: six>=1.5 in ./.venv/lib/python3.13/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", - "Collecting soupsieve>1.2 (from beautifulsoup4->gdown)\n", - " Using cached soupsieve-2.6-py3-none-any.whl.metadata (4.6 kB)\n", - "Collecting charset-normalizer<4,>=2 (from requests[socks]->gdown)\n", - " Downloading charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl.metadata (35 kB)\n", - "Collecting idna<4,>=2.5 (from requests[socks]->gdown)\n", - " Using cached idna-3.10-py3-none-any.whl.metadata (10 kB)\n", - "Collecting urllib3<3,>=1.21.1 (from requests[socks]->gdown)\n", - " Downloading urllib3-2.3.0-py3-none-any.whl.metadata (6.5 kB)\n", - "Collecting certifi>=2017.4.17 (from requests[socks]->gdown)\n", - " Downloading certifi-2024.12.14-py3-none-any.whl.metadata (2.3 kB)\n", - "Collecting PySocks!=1.5.7,>=1.5.6 (from requests[socks]->gdown)\n", - " Using cached PySocks-1.7.1-py3-none-any.whl.metadata (13 kB)\n", - "Using cached gdown-5.2.0-py3-none-any.whl (18 kB)\n", - "Using cached beautifulsoup4-4.12.3-py3-none-any.whl (147 kB)\n", - "Using cached filelock-3.16.1-py3-none-any.whl (16 kB)\n", - "Using cached tqdm-4.67.1-py3-none-any.whl (78 kB)\n", - "Downloading certifi-2024.12.14-py3-none-any.whl (164 kB)\n", - "Downloading charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl (195 kB)\n", - "Using cached idna-3.10-py3-none-any.whl (70 kB)\n", - "Using cached PySocks-1.7.1-py3-none-any.whl (16 kB)\n", - "Using cached soupsieve-2.6-py3-none-any.whl (36 kB)\n", - "Downloading urllib3-2.3.0-py3-none-any.whl (128 kB)\n", - "Using cached requests-2.32.3-py3-none-any.whl (64 kB)\n", - "Installing collected packages: urllib3, tqdm, soupsieve, PySocks, idna, filelock, charset-normalizer, certifi, requests, beautifulsoup4, gdown\n", - "Successfully installed PySocks-1.7.1 beautifulsoup4-4.12.3 certifi-2024.12.14 charset-normalizer-3.4.1 filelock-3.16.1 gdown-5.2.0 idna-3.10 requests-2.32.3 soupsieve-2.6 tqdm-4.67.1 urllib3-2.3.0\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "######## Installations - BE SURE TO MAKE YOUR OWN LOCAL VENV FIRST\n", - "\n", - "%pip install gdown pandas" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "KgzveXyAp35v" - }, - "outputs": [], - "source": [ - "######## Imports\n", - "\n", - "import pandas as pd\n", - "import gdown\n", - "import tarfile\n", - "import os\n", - "import json\n", - "from datetime import datetime\n", - "import csv" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "iwcBxgxgqES8", - "outputId": "4af8de17-57bd-4857-f49c-2ee5d39ed248" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "'./longmemeval_data/longmemeval_data.tar.gz' already exists, skipping download.\n", - "'longmemeval_oracle.json' already exists, so skipping extraction.\n" - ] - } - ], - "source": [ - "######## Download the eval dataset from the official Google Drive source\n", - "\n", - "file_id = '1zJgtYRFhOh5zDQzzatiddfjYhFSnyQ80'\n", - "url = f'https://drive.google.com/uc?id={file_id}'\n", - "folder_path = './longmemeval_data'\n", - "file_path = os.path.join(folder_path, 'longmemeval_data.tar.gz')\n", - "\n", - "# If it doesn't exist, create a \"/.longmemeval_data/\" directory\n", - "if not os.path.exists(folder_path):\n", - " os.makedirs(folder_path)\n", - "\n", - "# Download the compressed dataset\n", - "if not os.path.exists(file_path):\n", - " gdown.download(url, file_path, quiet=False)\n", - "else:\n", - " print(f\"'{file_path}' already exists, skipping download.\")\n", - "\n", - "# Extract the tar.gz\n", - "if not os.path.exists(os.path.join(folder_path, 'longmemeval_oracle.json')):\n", - " with tarfile.open(file_path, 'r:gz') as tar:\n", - " tar.extractall(path=folder_path)\n", - "else:\n", - " print(\"'longmemeval_oracle.json' already exists, so skipping extraction.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "_RjEZnk5v530" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
question_idquestion_typequestionanswerquestion_datehaystack_dateshaystack_session_idshaystack_sessionsanswer_session_ids
0gpt4_2655b836temporal-reasoningWhat was the first issue I had with my new car...GPS system not functioning correctly2023/04/10 (Mon) 23:07[2023/04/10 (Mon) 17:50, 2023/04/10 (Mon) 14:4...[answer_4be1b6b4_2, answer_4be1b6b4_3, answer_...[[{'role': 'user', 'content': 'I'm thinking of...[answer_4be1b6b4_2, answer_4be1b6b4_3, answer_...
1gpt4_2487a7cbtemporal-reasoningWhich event did I attend first, the 'Effective...'Data Analysis using Python' webinar2023/05/28 (Sun) 06:47[2023/05/28 (Sun) 21:04, 2023/05/28 (Sun) 07:17][answer_1c6b85ea_1, answer_1c6b85ea_2][[{'role': 'user', 'content': 'I'm trying to g...[answer_1c6b85ea_1, answer_1c6b85ea_2]
2gpt4_76048e76temporal-reasoningWhich vehicle did I take care of first in Febr...bike2023/03/10 (Fri) 23:15[2023/03/10 (Fri) 22:50, 2023/03/10 (Fri) 08:11][answer_b535969f_2, answer_b535969f_1][[{'role': 'user', 'content': 'I'm thinking of...[answer_b535969f_2, answer_b535969f_1]
3gpt4_2312f94ctemporal-reasoningWhich device did I got first, the Samsung Gala...Samsung Galaxy S222023/03/15 (Wed) 03:53[2023/03/15 (Wed) 00:56, 2023/03/15 (Wed) 10:31][answer_5328c3c2_1, answer_5328c3c2_2][[{'role': 'user', 'content': 'I'm planning a ...[answer_5328c3c2_1, answer_5328c3c2_2]
40bb5a684temporal-reasoningHow many days before the team meeting I was pr...7 days. 8 days (including the last day) is als...2023/01/13 (Fri) 19:39[2023/01/13 (Fri) 18:07, 2023/01/13 (Fri) 21:38][answer_e936197f_1, answer_e936197f_2][[{'role': 'user', 'content': 'I'm preparing f...[answer_e936197f_1, answer_e936197f_2]
\n", - "
" - ], - "text/plain": [ - " question_id question_type \\\n", - "0 gpt4_2655b836 temporal-reasoning \n", - "1 gpt4_2487a7cb temporal-reasoning \n", - "2 gpt4_76048e76 temporal-reasoning \n", - "3 gpt4_2312f94c temporal-reasoning \n", - "4 0bb5a684 temporal-reasoning \n", - "\n", - " question \\\n", - "0 What was the first issue I had with my new car... \n", - "1 Which event did I attend first, the 'Effective... \n", - "2 Which vehicle did I take care of first in Febr... \n", - "3 Which device did I got first, the Samsung Gala... \n", - "4 How many days before the team meeting I was pr... \n", - "\n", - " answer question_date \\\n", - "0 GPS system not functioning correctly 2023/04/10 (Mon) 23:07 \n", - "1 'Data Analysis using Python' webinar 2023/05/28 (Sun) 06:47 \n", - "2 bike 2023/03/10 (Fri) 23:15 \n", - "3 Samsung Galaxy S22 2023/03/15 (Wed) 03:53 \n", - "4 7 days. 8 days (including the last day) is als... 2023/01/13 (Fri) 19:39 \n", - "\n", - " haystack_dates \\\n", - "0 [2023/04/10 (Mon) 17:50, 2023/04/10 (Mon) 14:4... \n", - "1 [2023/05/28 (Sun) 21:04, 2023/05/28 (Sun) 07:17] \n", - "2 [2023/03/10 (Fri) 22:50, 2023/03/10 (Fri) 08:11] \n", - "3 [2023/03/15 (Wed) 00:56, 2023/03/15 (Wed) 10:31] \n", - "4 [2023/01/13 (Fri) 18:07, 2023/01/13 (Fri) 21:38] \n", - "\n", - " haystack_session_ids \\\n", - "0 [answer_4be1b6b4_2, answer_4be1b6b4_3, answer_... \n", - "1 [answer_1c6b85ea_1, answer_1c6b85ea_2] \n", - "2 [answer_b535969f_2, answer_b535969f_1] \n", - "3 [answer_5328c3c2_1, answer_5328c3c2_2] \n", - "4 [answer_e936197f_1, answer_e936197f_2] \n", - "\n", - " haystack_sessions \\\n", - "0 [[{'role': 'user', 'content': 'I'm thinking of... \n", - "1 [[{'role': 'user', 'content': 'I'm trying to g... \n", - "2 [[{'role': 'user', 'content': 'I'm thinking of... \n", - "3 [[{'role': 'user', 'content': 'I'm planning a ... \n", - "4 [[{'role': 'user', 'content': 'I'm preparing f... \n", - "\n", - " answer_session_ids \n", - "0 [answer_4be1b6b4_2, answer_4be1b6b4_3, answer_... \n", - "1 [answer_1c6b85ea_1, answer_1c6b85ea_2] \n", - "2 [answer_b535969f_2, answer_b535969f_1] \n", - "3 [answer_5328c3c2_1, answer_5328c3c2_2] \n", - "4 [answer_e936197f_1, answer_e936197f_2] " - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "######## Load the eval dataset\n", - "\n", - "lme_dataset_option = os.path.join(folder_path, 'longmemeval_oracle.json') # Can be _oracle, _s, or _m\n", - "lme_dataset_df = pd.read_json(lme_dataset_option)\n", - "lme_dataset_df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "######## Method to save all of the snippets (or only firsts/lasts) of the specified multi-sessions to a CSV file\n", - "\n", - "\n", - "def snippetize_lme_dataset(lme_filename, max_num_previous_messages=5):\n", - " \"\"\"\n", - " Creates a csv where each row is a \"snippet\" from longmemeval. A snippet is a message and set of previous messages.\n", - " \"\"\"\n", - "\n", - " lme_dataset_option = os.path.join(folder_path, lme_filename)\n", - " lme_dataset_df = pd.read_json(lme_dataset_option)\n", - "\n", - " all_snippets = []\n", - " for index, row in lme_dataset_df.iterrows():\n", - "\n", - " question_id = row['question_id']\n", - "\n", - " # Extract the haystack_sessions and dates\n", - " sessions = row['haystack_sessions']\n", - " session_dates = row['haystack_dates']\n", - "\n", - " # Combine into list of dictionaries\n", - " sessions_data = [\n", - " {\n", - " \"session\": session,\n", - " \"date\": datetime.strptime(date, \"%Y/%m/%d (%a) %H:%M\")\n", - " } \n", - " for session, date in zip(sessions, session_dates)\n", - " ]\n", - "\n", - " # Sort by date from earliest to latest\n", - " sessions_data.sort(key=lambda x: x[\"date\"])\n", - "\n", - "\n", - " all_snippets_this_session = []\n", - "\n", - " message_index_across_sessions = 0\n", - " for session_index, session_and_date in enumerate(sessions_data):\n", - " for message_index_within_session, message in enumerate(session_and_date[\"session\"]):\n", - " \n", - " num_previous_messages = min(max_num_previous_messages, message_index_across_sessions)\n", - " previous_snippets = all_snippets_this_session[message_index_across_sessions-num_previous_messages:]\n", - " previous_messages_only = [{\"role\":previous_snippet[\"message\"][\"role\"], \"content\":previous_snippet[\"message\"][\"content\"]} for previous_snippet in previous_snippets]\n", - "\n", - " snippet = {\n", - " \"question_id\": question_id,\n", - " \"question_type\": row[\"question_type\"],\n", - " \"multisession_index\": index,\n", - " \"session_index\": session_index,\n", - " \"message_index_within_session\": message_index_within_session,\n", - " \"message_index_across_sessions\": message_index_across_sessions,\n", - " \"session_date\": session_and_date[\"date\"],\n", - " \"message\": message,\n", - " \"previous_messages\": previous_messages_only,\n", - " \"num_previous_messages\": num_previous_messages,\n", - " }\n", - "\n", - " if lme_filename == \"longmemeval_oracle.json\":\n", - " snippet[\"message_has_answer\"] = message[\"has_answer\"]\n", - "\n", - " all_snippets_this_session.append(snippet)\n", - " message_index_across_sessions += 1\n", - "\n", - " all_snippets.extend(all_snippets_this_session)\n", - "\n", - " \n", - "\n", - " snippetized_folder = os.path.join(folder_path, \"snippetized_data\")\n", - " if not os.path.exists(snippetized_folder):\n", - " os.makedirs(snippetized_folder)\n", - "\n", - " filename = lme_filename.replace(\".json\", \"_snippetized.csv\")\n", - " filepath = os.path.join(snippetized_folder, filename)\n", - " \n", - " with open(filepath, \"w\", newline=\"\") as csvfile:\n", - " writer = csv.DictWriter(csvfile, fieldnames=all_snippets[0].keys())\n", - " writer.writeheader()\n", - " for snippet in all_snippets:\n", - " snippet['message'] = json.dumps(snippet['message'])\n", - " snippet['previous_messages'] = json.dumps(snippet['previous_messages'])\n", - " writer.writerow(snippet)\n", - "\n", - "\n", - "\n", - "\n", - "def snippetize_and_check(lme_filename):\n", - " folder_path = './longmemeval_data/snippetized_data'\n", - " file_path = os.path.join(folder_path, lme_filename.replace(\".json\", \"_snippetized.csv\"))\n", - " if not os.path.exists(file_path):\n", - " print(f\"Snippetizing {lme_filename}...\")\n", - " snippetize_lme_dataset(lme_filename)\n", - " else:\n", - " print(f\"Skipping snippetization for {lme_filename} because it already exists.\")\n", - "\n", - " # Check first few rows of the csv\n", - " df = pd.read_csv(file_path)\n", - " display(df.head(10))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Snippetizing longmemeval_oracle.json...\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
question_idquestion_typemultisession_indexsession_indexmessage_index_within_sessionmessage_index_across_sessionssession_datemessageprevious_messagesnum_previous_messagesmessage_has_answer
0gpt4_2655b836temporal-reasoning00002023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I'm thinking of g...[]0False
1gpt4_2655b836temporal-reasoning00112023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"Choosing the...[{\"role\": \"user\", \"content\": \"I'm thinking of ...1False
2gpt4_2655b836temporal-reasoning00222023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I've been doing s...[{\"role\": \"user\", \"content\": \"I'm thinking of ...2True
3gpt4_2655b836temporal-reasoning00332023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"That's great...[{\"role\": \"user\", \"content\": \"I'm thinking of ...3False
4gpt4_2655b836temporal-reasoning00442023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I'll definitely a...[{\"role\": \"user\", \"content\": \"I'm thinking of ...4False
5gpt4_2655b836temporal-reasoning00552023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"Advanced pai...[{\"role\": \"user\", \"content\": \"I'm thinking of ...5False
6gpt4_2655b836temporal-reasoning00662023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I'll definitely a...[{\"role\": \"assistant\", \"content\": \"Choosing th...5False
7gpt4_2655b836temporal-reasoning00772023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"Congratulati...[{\"role\": \"user\", \"content\": \"I've been doing ...5False
8gpt4_2655b836temporal-reasoning00882023-04-10 14:47:00{\"role\": \"user\", \"content\": \"That's really hel...[{\"role\": \"assistant\", \"content\": \"That's grea...5False
9gpt4_2655b836temporal-reasoning00992023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"Congratulati...[{\"role\": \"user\", \"content\": \"I'll definitely ...5False
\n", - "
" - ], - "text/plain": [ - " question_id question_type multisession_index session_index \\\n", - "0 gpt4_2655b836 temporal-reasoning 0 0 \n", - "1 gpt4_2655b836 temporal-reasoning 0 0 \n", - "2 gpt4_2655b836 temporal-reasoning 0 0 \n", - "3 gpt4_2655b836 temporal-reasoning 0 0 \n", - "4 gpt4_2655b836 temporal-reasoning 0 0 \n", - "5 gpt4_2655b836 temporal-reasoning 0 0 \n", - "6 gpt4_2655b836 temporal-reasoning 0 0 \n", - "7 gpt4_2655b836 temporal-reasoning 0 0 \n", - "8 gpt4_2655b836 temporal-reasoning 0 0 \n", - "9 gpt4_2655b836 temporal-reasoning 0 0 \n", - "\n", - " message_index_within_session message_index_across_sessions \\\n", - "0 0 0 \n", - "1 1 1 \n", - "2 2 2 \n", - "3 3 3 \n", - "4 4 4 \n", - "5 5 5 \n", - "6 6 6 \n", - "7 7 7 \n", - "8 8 8 \n", - "9 9 9 \n", - "\n", - " session_date message \\\n", - "0 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'm thinking of g... \n", - "1 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Choosing the... \n", - "2 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I've been doing s... \n", - "3 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"That's great... \n", - "4 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'll definitely a... \n", - "5 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Advanced pai... \n", - "6 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'll definitely a... \n", - "7 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Congratulati... \n", - "8 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"That's really hel... \n", - "9 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Congratulati... \n", - "\n", - " previous_messages num_previous_messages \\\n", - "0 [] 0 \n", - "1 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 1 \n", - "2 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 2 \n", - "3 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 3 \n", - "4 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 4 \n", - "5 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 5 \n", - "6 [{\"role\": \"assistant\", \"content\": \"Choosing th... 5 \n", - "7 [{\"role\": \"user\", \"content\": \"I've been doing ... 5 \n", - "8 [{\"role\": \"assistant\", \"content\": \"That's grea... 5 \n", - "9 [{\"role\": \"user\", \"content\": \"I'll definitely ... 5 \n", - "\n", - " message_has_answer \n", - "0 False \n", - "1 False \n", - "2 True \n", - "3 False \n", - "4 False \n", - "5 False \n", - "6 False \n", - "7 False \n", - "8 False \n", - "9 False " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "lme_filename = \"longmemeval_oracle.json\"\n", - "snippetize_and_check(lme_filename)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Snippetizing longmemeval_s.json...\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
question_idquestion_typemultisession_indexsession_indexmessage_index_within_sessionmessage_index_across_sessionssession_datemessageprevious_messagesnum_previous_messages
0e47becbasingle-session-user00002023-05-20 02:21:00{\"role\": \"user\", \"content\": \"The farmer needs ...[]0
1e47becbasingle-session-user00112023-05-20 02:21:00{\"role\": \"assistant\", \"content\": \"To solve thi...[{\"role\": \"user\", \"content\": \"The farmer needs...1
2e47becbasingle-session-user01022023-05-20 02:57:00{\"role\": \"user\", \"content\": \"I'm trying to sta...[{\"role\": \"user\", \"content\": \"The farmer needs...2
3e47becbasingle-session-user01132023-05-20 02:57:00{\"role\": \"assistant\", \"content\": \"Congratulati...[{\"role\": \"user\", \"content\": \"The farmer needs...3
4e47becbasingle-session-user01242023-05-20 02:57:00{\"role\": \"user\", \"content\": \"I've been doing s...[{\"role\": \"user\", \"content\": \"The farmer needs...4
5e47becbasingle-session-user01352023-05-20 02:57:00{\"role\": \"assistant\", \"content\": \"Yoga is an e...[{\"role\": \"user\", \"content\": \"The farmer needs...5
6e47becbasingle-session-user01462023-05-20 02:57:00{\"role\": \"user\", \"content\": \"That's really hel...[{\"role\": \"assistant\", \"content\": \"To solve th...5
7e47becbasingle-session-user01572023-05-20 02:57:00{\"role\": \"assistant\", \"content\": \"Foam rolling...[{\"role\": \"user\", \"content\": \"I'm trying to st...5
8e47becbasingle-session-user01682023-05-20 02:57:00{\"role\": \"user\", \"content\": \"I've also been tr...[{\"role\": \"assistant\", \"content\": \"Congratulat...5
9e47becbasingle-session-user01792023-05-20 02:57:00{\"role\": \"assistant\", \"content\": \"Monitoring y...[{\"role\": \"user\", \"content\": \"I've been doing ...5
\n", - "
" - ], - "text/plain": [ - " question_id question_type multisession_index session_index \\\n", - "0 e47becba single-session-user 0 0 \n", - "1 e47becba single-session-user 0 0 \n", - "2 e47becba single-session-user 0 1 \n", - "3 e47becba single-session-user 0 1 \n", - "4 e47becba single-session-user 0 1 \n", - "5 e47becba single-session-user 0 1 \n", - "6 e47becba single-session-user 0 1 \n", - "7 e47becba single-session-user 0 1 \n", - "8 e47becba single-session-user 0 1 \n", - "9 e47becba single-session-user 0 1 \n", - "\n", - " message_index_within_session message_index_across_sessions \\\n", - "0 0 0 \n", - "1 1 1 \n", - "2 0 2 \n", - "3 1 3 \n", - "4 2 4 \n", - "5 3 5 \n", - "6 4 6 \n", - "7 5 7 \n", - "8 6 8 \n", - "9 7 9 \n", - "\n", - " session_date message \\\n", - "0 2023-05-20 02:21:00 {\"role\": \"user\", \"content\": \"The farmer needs ... \n", - "1 2023-05-20 02:21:00 {\"role\": \"assistant\", \"content\": \"To solve thi... \n", - "2 2023-05-20 02:57:00 {\"role\": \"user\", \"content\": \"I'm trying to sta... \n", - "3 2023-05-20 02:57:00 {\"role\": \"assistant\", \"content\": \"Congratulati... \n", - "4 2023-05-20 02:57:00 {\"role\": \"user\", \"content\": \"I've been doing s... \n", - "5 2023-05-20 02:57:00 {\"role\": \"assistant\", \"content\": \"Yoga is an e... \n", - "6 2023-05-20 02:57:00 {\"role\": \"user\", \"content\": \"That's really hel... \n", - "7 2023-05-20 02:57:00 {\"role\": \"assistant\", \"content\": \"Foam rolling... \n", - "8 2023-05-20 02:57:00 {\"role\": \"user\", \"content\": \"I've also been tr... \n", - "9 2023-05-20 02:57:00 {\"role\": \"assistant\", \"content\": \"Monitoring y... \n", - "\n", - " previous_messages num_previous_messages \n", - "0 [] 0 \n", - "1 [{\"role\": \"user\", \"content\": \"The farmer needs... 1 \n", - "2 [{\"role\": \"user\", \"content\": \"The farmer needs... 2 \n", - "3 [{\"role\": \"user\", \"content\": \"The farmer needs... 3 \n", - "4 [{\"role\": \"user\", \"content\": \"The farmer needs... 4 \n", - "5 [{\"role\": \"user\", \"content\": \"The farmer needs... 5 \n", - "6 [{\"role\": \"assistant\", \"content\": \"To solve th... 5 \n", - "7 [{\"role\": \"user\", \"content\": \"I'm trying to st... 5 \n", - "8 [{\"role\": \"assistant\", \"content\": \"Congratulat... 5 \n", - "9 [{\"role\": \"user\", \"content\": \"I've been doing ... 5 " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "lme_filename = \"longmemeval_s.json\"\n", - "snippetize_and_check(lme_filename)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Snippetizing longmemeval_m.json...\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
question_idquestion_typemultisession_indexsession_indexmessage_index_within_sessionmessage_index_across_sessionssession_datemessageprevious_messagesnum_previous_messages
07161e7e2single-session-assistant00002023-05-20 00:04:00{\"role\": \"user\", \"content\": \"Can you provide t...[]0
17161e7e2single-session-assistant00112023-05-20 00:04:00{\"role\": \"assistant\", \"content\": \"Yes, here ar...[{\"role\": \"user\", \"content\": \"Can you provide ...1
27161e7e2single-session-assistant00222023-05-20 00:04:00{\"role\": \"user\", \"content\": \"Wow, it's amazing...[{\"role\": \"user\", \"content\": \"Can you provide ...2
37161e7e2single-session-assistant00332023-05-20 00:04:00{\"role\": \"assistant\", \"content\": \"Yes, achievi...[{\"role\": \"user\", \"content\": \"Can you provide ...3
47161e7e2single-session-assistant00442023-05-20 00:04:00{\"role\": \"user\", \"content\": \"It's crazy to thi...[{\"role\": \"user\", \"content\": \"Can you provide ...4
57161e7e2single-session-assistant00552023-05-20 00:04:00{\"role\": \"assistant\", \"content\": \"Astronauts u...[{\"role\": \"user\", \"content\": \"Can you provide ...5
67161e7e2single-session-assistant01062023-05-20 00:16:00{\"role\": \"user\", \"content\": \"What about aliena...[{\"role\": \"assistant\", \"content\": \"Yes, here a...5
77161e7e2single-session-assistant01172023-05-20 00:16:00{\"role\": \"assistant\", \"content\": \"Alienation i...[{\"role\": \"user\", \"content\": \"Wow, it's amazin...5
87161e7e2single-session-assistant01282023-05-20 00:16:00{\"role\": \"user\", \"content\": \"Why you did not m...[{\"role\": \"assistant\", \"content\": \"Yes, achiev...5
97161e7e2single-session-assistant01392023-05-20 00:16:00{\"role\": \"assistant\", \"content\": \"I apologize ...[{\"role\": \"user\", \"content\": \"It's crazy to th...5
\n", - "
" - ], - "text/plain": [ - " question_id question_type multisession_index session_index \\\n", - "0 7161e7e2 single-session-assistant 0 0 \n", - "1 7161e7e2 single-session-assistant 0 0 \n", - "2 7161e7e2 single-session-assistant 0 0 \n", - "3 7161e7e2 single-session-assistant 0 0 \n", - "4 7161e7e2 single-session-assistant 0 0 \n", - "5 7161e7e2 single-session-assistant 0 0 \n", - "6 7161e7e2 single-session-assistant 0 1 \n", - "7 7161e7e2 single-session-assistant 0 1 \n", - "8 7161e7e2 single-session-assistant 0 1 \n", - "9 7161e7e2 single-session-assistant 0 1 \n", - "\n", - " message_index_within_session message_index_across_sessions \\\n", - "0 0 0 \n", - "1 1 1 \n", - "2 2 2 \n", - "3 3 3 \n", - "4 4 4 \n", - "5 5 5 \n", - "6 0 6 \n", - "7 1 7 \n", - "8 2 8 \n", - "9 3 9 \n", - "\n", - " session_date message \\\n", - "0 2023-05-20 00:04:00 {\"role\": \"user\", \"content\": \"Can you provide t... \n", - "1 2023-05-20 00:04:00 {\"role\": \"assistant\", \"content\": \"Yes, here ar... \n", - "2 2023-05-20 00:04:00 {\"role\": \"user\", \"content\": \"Wow, it's amazing... \n", - "3 2023-05-20 00:04:00 {\"role\": \"assistant\", \"content\": \"Yes, achievi... \n", - "4 2023-05-20 00:04:00 {\"role\": \"user\", \"content\": \"It's crazy to thi... \n", - "5 2023-05-20 00:04:00 {\"role\": \"assistant\", \"content\": \"Astronauts u... \n", - "6 2023-05-20 00:16:00 {\"role\": \"user\", \"content\": \"What about aliena... \n", - "7 2023-05-20 00:16:00 {\"role\": \"assistant\", \"content\": \"Alienation i... \n", - "8 2023-05-20 00:16:00 {\"role\": \"user\", \"content\": \"Why you did not m... \n", - "9 2023-05-20 00:16:00 {\"role\": \"assistant\", \"content\": \"I apologize ... \n", - "\n", - " previous_messages num_previous_messages \n", - "0 [] 0 \n", - "1 [{\"role\": \"user\", \"content\": \"Can you provide ... 1 \n", - "2 [{\"role\": \"user\", \"content\": \"Can you provide ... 2 \n", - "3 [{\"role\": \"user\", \"content\": \"Can you provide ... 3 \n", - "4 [{\"role\": \"user\", \"content\": \"Can you provide ... 4 \n", - "5 [{\"role\": \"user\", \"content\": \"Can you provide ... 5 \n", - "6 [{\"role\": \"assistant\", \"content\": \"Yes, here a... 5 \n", - "7 [{\"role\": \"user\", \"content\": \"Wow, it's amazin... 5 \n", - "8 [{\"role\": \"assistant\", \"content\": \"Yes, achiev... 5 \n", - "9 [{\"role\": \"user\", \"content\": \"It's crazy to th... 5 " - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "lme_filename = \"longmemeval_m.json\"\n", - "snippetize_and_check(lme_filename)" - ] - } - ], - "metadata": { + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { "colab": { - "provenance": [] + "base_uri": "https://localhost:8080/" }, - "kernelspec": { - "display_name": "Python (.venv)", - "language": "python", - "name": ".venv" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.0" - } + "id": "i9i9uUZ3pWQE", + "outputId": "84404bb8-5841-4f2f-dd87-7f909c6e95aa" + }, + "outputs": [], + "source": [ + "######## Installations - BE SURE TO MAKE YOUR OWN LOCAL VENV FIRST\n", + "\n", + "%pip install gdown pandas" + ] }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "KgzveXyAp35v" + }, + "outputs": [], + "source": [ + "######## Imports\n", + "\n", + "import csv\n", + "import json\n", + "import os\n", + "import tarfile\n", + "from datetime import datetime\n", + "\n", + "import gdown\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "iwcBxgxgqES8", + "outputId": "4af8de17-57bd-4857-f49c-2ee5d39ed248" + }, + "outputs": [], + "source": [ + "######## Download the eval dataset from the official Google Drive source\n", + "\n", + "file_id = '1zJgtYRFhOh5zDQzzatiddfjYhFSnyQ80'\n", + "url = f'https://drive.google.com/uc?id={file_id}'\n", + "folder_path = './longmemeval_data'\n", + "file_path = os.path.join(folder_path, 'longmemeval_data.tar.gz')\n", + "\n", + "# If it doesn't exist, create a \"/.longmemeval_data/\" directory\n", + "if not os.path.exists(folder_path):\n", + " os.makedirs(folder_path)\n", + "\n", + "# Download the compressed dataset\n", + "if not os.path.exists(file_path):\n", + " gdown.download(url, file_path, quiet=False)\n", + "else:\n", + " print(f\"'{file_path}' already exists, skipping download.\")\n", + "\n", + "# Extract the tar.gz\n", + "if not os.path.exists(os.path.join(folder_path, 'longmemeval_oracle.json')):\n", + " with tarfile.open(file_path, 'r:gz') as tar:\n", + " tar.extractall(path=folder_path)\n", + "else:\n", + " print(\"'longmemeval_oracle.json' already exists, so skipping extraction.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_RjEZnk5v530" + }, + "outputs": [], + "source": [ + "######## Load the eval dataset\n", + "\n", + "lme_dataset_option = os.path.join(\n", + " folder_path, 'longmemeval_oracle.json'\n", + ") # Can be _oracle, _s, or _m\n", + "lme_dataset_df = pd.read_json(lme_dataset_option)\n", + "lme_dataset_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "######## Method to save all of the snippets (or only firsts/lasts) of the specified multi-sessions to a CSV file\n", + "\n", + "\n", + "def snippetize_lme_dataset(lme_filename, max_num_previous_messages=5):\n", + " \"\"\"\n", + " Creates a csv where each row is a \"snippet\" from longmemeval. A snippet is a message and set of previous messages.\n", + " \"\"\"\n", + "\n", + " lme_dataset_option = os.path.join(folder_path, lme_filename)\n", + " lme_dataset_df = pd.read_json(lme_dataset_option)\n", + "\n", + " all_snippets = []\n", + " for index, row in lme_dataset_df.iterrows():\n", + " question_id = row['question_id']\n", + "\n", + " # Extract the haystack_sessions and dates\n", + " sessions = row['haystack_sessions']\n", + " session_dates = row['haystack_dates']\n", + "\n", + " # Combine into list of dictionaries\n", + " sessions_data = [\n", + " {'session': session, 'date': datetime.strptime(date, '%Y/%m/%d (%a) %H:%M')}\n", + " for session, date in zip(sessions, session_dates)\n", + " ]\n", + "\n", + " # Sort by date from earliest to latest\n", + " sessions_data.sort(key=lambda x: x['date'])\n", + "\n", + " all_snippets_this_session = []\n", + "\n", + " message_index_across_sessions = 0\n", + " for session_index, session_and_date in enumerate(sessions_data):\n", + " for message_index_within_session, message in enumerate(session_and_date['session']):\n", + " num_previous_messages = min(\n", + " max_num_previous_messages, message_index_across_sessions\n", + " )\n", + " previous_snippets = all_snippets_this_session[\n", + " message_index_across_sessions - num_previous_messages:\n", + " ]\n", + " previous_messages_only = [\n", + " {\n", + " 'role': previous_snippet['message']['role'],\n", + " 'content': previous_snippet['message']['content'],\n", + " }\n", + " for previous_snippet in previous_snippets\n", + " ]\n", + "\n", + " snippet = {\n", + " 'question_id': question_id,\n", + " 'question_type': row['question_type'],\n", + " 'multisession_index': index,\n", + " 'session_index': session_index,\n", + " 'message_index_within_session': message_index_within_session,\n", + " 'message_index_across_sessions': message_index_across_sessions,\n", + " 'session_date': session_and_date['date'],\n", + " 'message': message,\n", + " 'previous_messages': previous_messages_only,\n", + " 'num_previous_messages': num_previous_messages,\n", + " }\n", + "\n", + " if lme_filename == 'longmemeval_oracle.json':\n", + " snippet['message_has_answer'] = message['has_answer']\n", + "\n", + " all_snippets_this_session.append(snippet)\n", + " message_index_across_sessions += 1\n", + "\n", + " all_snippets.extend(all_snippets_this_session)\n", + "\n", + " snippetized_folder = os.path.join(folder_path, 'snippetized_data')\n", + " if not os.path.exists(snippetized_folder):\n", + " os.makedirs(snippetized_folder)\n", + "\n", + " filename = lme_filename.replace('.json', '_snippetized.csv')\n", + " filepath = os.path.join(snippetized_folder, filename)\n", + "\n", + " with open(filepath, 'w', newline='') as csvfile:\n", + " writer = csv.DictWriter(csvfile, fieldnames=all_snippets[0].keys())\n", + " writer.writeheader()\n", + " for snippet in all_snippets:\n", + " snippet['message'] = json.dumps(snippet['message'])\n", + " snippet['previous_messages'] = json.dumps(snippet['previous_messages'])\n", + " writer.writerow(snippet)\n", + "\n", + "\n", + "def snippetize_and_check(lme_filename):\n", + " folder_path = './longmemeval_data/snippetized_data'\n", + " file_path = os.path.join(folder_path, lme_filename.replace('.json', '_snippetized.csv'))\n", + " if not os.path.exists(file_path):\n", + " print(f'Snippetizing {lme_filename}...')\n", + " snippetize_lme_dataset(lme_filename)\n", + " else:\n", + " print(f'Skipping snippetization for {lme_filename} because it already exists.')\n", + "\n", + " # Check first few rows of the csv\n", + " df = pd.read_csv(file_path)\n", + " display(df.head(10))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-20T17:17:33.985547Z", + "start_time": "2025-03-20T17:17:33.569496Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Skipping snippetization for longmemeval_oracle.json because it already exists.\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
question_idquestion_typemultisession_indexsession_indexmessage_index_within_sessionmessage_index_across_sessionssession_datemessageprevious_messagesnum_previous_messagesmessage_has_answer
0gpt4_2655b836temporal-reasoning00002023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I'm thinking of g...[]0False
1gpt4_2655b836temporal-reasoning00112023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"Choosing the...[{\"role\": \"user\", \"content\": \"I'm thinking of ...1False
2gpt4_2655b836temporal-reasoning00222023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I've been doing s...[{\"role\": \"user\", \"content\": \"I'm thinking of ...2True
3gpt4_2655b836temporal-reasoning00332023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"That's great...[{\"role\": \"user\", \"content\": \"I'm thinking of ...3False
4gpt4_2655b836temporal-reasoning00442023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I'll definitely a...[{\"role\": \"user\", \"content\": \"I'm thinking of ...4False
5gpt4_2655b836temporal-reasoning00552023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"Advanced pai...[{\"role\": \"user\", \"content\": \"I'm thinking of ...5False
6gpt4_2655b836temporal-reasoning00662023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I'll definitely a...[{\"role\": \"assistant\", \"content\": \"Choosing th...5False
7gpt4_2655b836temporal-reasoning00772023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"Congratulati...[{\"role\": \"user\", \"content\": \"I've been doing ...5False
8gpt4_2655b836temporal-reasoning00882023-04-10 14:47:00{\"role\": \"user\", \"content\": \"That's really hel...[{\"role\": \"assistant\", \"content\": \"That's grea...5False
9gpt4_2655b836temporal-reasoning00992023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"Congratulati...[{\"role\": \"user\", \"content\": \"I'll definitely ...5False
\n", + "
" + ], + "text/plain": [ + " question_id question_type multisession_index session_index \\\n", + "0 gpt4_2655b836 temporal-reasoning 0 0 \n", + "1 gpt4_2655b836 temporal-reasoning 0 0 \n", + "2 gpt4_2655b836 temporal-reasoning 0 0 \n", + "3 gpt4_2655b836 temporal-reasoning 0 0 \n", + "4 gpt4_2655b836 temporal-reasoning 0 0 \n", + "5 gpt4_2655b836 temporal-reasoning 0 0 \n", + "6 gpt4_2655b836 temporal-reasoning 0 0 \n", + "7 gpt4_2655b836 temporal-reasoning 0 0 \n", + "8 gpt4_2655b836 temporal-reasoning 0 0 \n", + "9 gpt4_2655b836 temporal-reasoning 0 0 \n", + "\n", + " message_index_within_session message_index_across_sessions \\\n", + "0 0 0 \n", + "1 1 1 \n", + "2 2 2 \n", + "3 3 3 \n", + "4 4 4 \n", + "5 5 5 \n", + "6 6 6 \n", + "7 7 7 \n", + "8 8 8 \n", + "9 9 9 \n", + "\n", + " session_date message \\\n", + "0 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'm thinking of g... \n", + "1 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Choosing the... \n", + "2 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I've been doing s... \n", + "3 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"That's great... \n", + "4 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'll definitely a... \n", + "5 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Advanced pai... \n", + "6 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'll definitely a... \n", + "7 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Congratulati... \n", + "8 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"That's really hel... \n", + "9 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Congratulati... \n", + "\n", + " previous_messages num_previous_messages \\\n", + "0 [] 0 \n", + "1 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 1 \n", + "2 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 2 \n", + "3 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 3 \n", + "4 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 4 \n", + "5 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 5 \n", + "6 [{\"role\": \"assistant\", \"content\": \"Choosing th... 5 \n", + "7 [{\"role\": \"user\", \"content\": \"I've been doing ... 5 \n", + "8 [{\"role\": \"assistant\", \"content\": \"That's grea... 5 \n", + "9 [{\"role\": \"user\", \"content\": \"I'll definitely ... 5 \n", + "\n", + " message_has_answer \n", + "0 False \n", + "1 False \n", + "2 True \n", + "3 False \n", + "4 False \n", + "5 False \n", + "6 False \n", + "7 False \n", + "8 False \n", + "9 False " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "lme_filename = 'longmemeval_oracle.json'\n", + "snippetize_and_check(lme_filename)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lme_filename = 'longmemeval_s.json'\n", + "snippetize_and_check(lme_filename)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lme_filename = 'longmemeval_m.json'\n", + "snippetize_and_check(lme_filename)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/tests/evals/data/LongMemEval_mini_dataset_loading.ipynb b/tests/evals/data/LongMemEval_mini_dataset_loading.ipynb index f605f556..73b3415d 100644 --- a/tests/evals/data/LongMemEval_mini_dataset_loading.ipynb +++ b/tests/evals/data/LongMemEval_mini_dataset_loading.ipynb @@ -1,2124 +1,2175 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "i9i9uUZ3pWQE", - "outputId": "84404bb8-5841-4f2f-dd87-7f909c6e95aa" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: pandas in ./.venv/lib/python3.13/site-packages (2.2.3)\n", - "Requirement already satisfied: graphiti-core in ./.venv/lib/python3.13/site-packages (0.5.1)\n", - "Requirement already satisfied: numpy>=1.26.0 in ./.venv/lib/python3.13/site-packages (from pandas) (2.2.1)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in ./.venv/lib/python3.13/site-packages (from pandas) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in ./.venv/lib/python3.13/site-packages (from pandas) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in ./.venv/lib/python3.13/site-packages (from pandas) (2024.2)\n", - "Requirement already satisfied: diskcache<6.0.0,>=5.6.3 in ./.venv/lib/python3.13/site-packages (from graphiti-core) (5.6.3)\n", - "Requirement already satisfied: neo4j<6.0.0,>=5.23.0 in ./.venv/lib/python3.13/site-packages (from graphiti-core) (5.27.0)\n", - "Requirement already satisfied: openai<2.0.0,>=1.53.0 in ./.venv/lib/python3.13/site-packages (from graphiti-core) (1.59.3)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.8.2 in ./.venv/lib/python3.13/site-packages (from graphiti-core) (2.10.4)\n", - "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.1 in ./.venv/lib/python3.13/site-packages (from graphiti-core) (1.0.1)\n", - "Requirement already satisfied: tenacity==9.0.0 in ./.venv/lib/python3.13/site-packages (from graphiti-core) (9.0.0)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in ./.venv/lib/python3.13/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (4.7.0)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in ./.venv/lib/python3.13/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (1.9.0)\n", - "Requirement already satisfied: httpx<1,>=0.23.0 in ./.venv/lib/python3.13/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (0.28.1)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in ./.venv/lib/python3.13/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (0.8.2)\n", - "Requirement already satisfied: sniffio in ./.venv/lib/python3.13/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (1.3.1)\n", - "Requirement already satisfied: tqdm>4 in ./.venv/lib/python3.13/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (4.67.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.11 in ./.venv/lib/python3.13/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (4.12.2)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in ./.venv/lib/python3.13/site-packages (from pydantic<3.0.0,>=2.8.2->graphiti-core) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.2 in ./.venv/lib/python3.13/site-packages (from pydantic<3.0.0,>=2.8.2->graphiti-core) (2.27.2)\n", - "Requirement already satisfied: six>=1.5 in ./.venv/lib/python3.13/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n", - "Requirement already satisfied: idna>=2.8 in ./.venv/lib/python3.13/site-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.53.0->graphiti-core) (3.10)\n", - "Requirement already satisfied: certifi in ./.venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.53.0->graphiti-core) (2024.12.14)\n", - "Requirement already satisfied: httpcore==1.* in ./.venv/lib/python3.13/site-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.53.0->graphiti-core) (1.0.7)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in ./.venv/lib/python3.13/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai<2.0.0,>=1.53.0->graphiti-core) (0.14.0)\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "######## Installations - BE SURE TO MAKE YOUR OWN LOCAL VENV FIRST\n", - "\n", - "%pip install pandas graphiti-core" - ] + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-20T16:16:43.665690Z", + "start_time": "2025-03-20T16:16:42.923920Z" }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "KgzveXyAp35v" - }, - "outputs": [], - "source": [ - "######## Imports\n", - "\n", - "import pandas as pd\n", - "import os\n", - "import json\n", - "import csv\n", - "from datetime import datetime\n", - "from utils import ingest_and_label_minidataset, dict_to_entity, entity_to_dict\n", - "from graphiti_core.llm_client import OpenAIClient\n", - "from graphiti_core.llm_client.config import LLMConfig\n", - "from graphiti_core.nodes import EntityNode" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "_RjEZnk5v530" - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
question_idquestion_typemultisession_indexsession_indexmessage_index_within_sessionmessage_index_across_sessionssession_datemessageprevious_messagesnum_previous_messagesmessage_has_answer
0gpt4_2655b836temporal-reasoning00002023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I'm thinking of g...[]0False
1gpt4_2655b836temporal-reasoning00112023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"Choosing the...[{\"role\": \"user\", \"content\": \"I'm thinking of ...1False
2gpt4_2655b836temporal-reasoning00222023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I've been doing s...[{\"role\": \"user\", \"content\": \"I'm thinking of ...2True
3gpt4_2655b836temporal-reasoning00332023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"That's great...[{\"role\": \"user\", \"content\": \"I'm thinking of ...3False
4gpt4_2655b836temporal-reasoning00442023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I'll definitely a...[{\"role\": \"user\", \"content\": \"I'm thinking of ...4False
\n", - "
" - ], - "text/plain": [ - " question_id question_type multisession_index session_index \\\n", - "0 gpt4_2655b836 temporal-reasoning 0 0 \n", - "1 gpt4_2655b836 temporal-reasoning 0 0 \n", - "2 gpt4_2655b836 temporal-reasoning 0 0 \n", - "3 gpt4_2655b836 temporal-reasoning 0 0 \n", - "4 gpt4_2655b836 temporal-reasoning 0 0 \n", - "\n", - " message_index_within_session message_index_across_sessions \\\n", - "0 0 0 \n", - "1 1 1 \n", - "2 2 2 \n", - "3 3 3 \n", - "4 4 4 \n", - "\n", - " session_date message \\\n", - "0 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'm thinking of g... \n", - "1 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Choosing the... \n", - "2 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I've been doing s... \n", - "3 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"That's great... \n", - "4 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'll definitely a... \n", - "\n", - " previous_messages num_previous_messages \\\n", - "0 [] 0 \n", - "1 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 1 \n", - "2 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 2 \n", - "3 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 3 \n", - "4 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 4 \n", - "\n", - " message_has_answer \n", - "0 False \n", - "1 False \n", - "2 True \n", - "3 False \n", - "4 False " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "######## Load the eval dataset\n", - "folder_path = './longmemeval_data/snippetized_data'\n", - "file_path = os.path.join(folder_path, 'longmemeval_oracle_snippetized.csv')\n", - "lme_dataset_df = pd.read_csv(file_path)\n", - "lme_dataset_df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "######## Methods\n", - "\n", - "MAX_NUM_PREVIOUS_MESSAGES = 5\n", - "\n", - "\n", - "def filter_for_zep_labelling(df):\n", - " \"\"\"\n", - " Filters the dataset for the snippets we want to use for Zep labelling.\n", - " \"\"\"\n", - "\n", - " # Filter to only rows where question_type = single_session_user\n", - " df = df[df['question_type'] == 'single-session-user']\n", - "\n", - " # Filter only where message_has_answer = True\n", - " df = df[df['message_has_answer'] == True]\n", - "\n", - " # Filter to only rows where num_previous_messages = 5\n", - " df = df[df['num_previous_messages'] == 5]\n", - "\n", - " # Limit to only 5 rows\n", - " df = df.head(1)\n", - "\n", - " return df\n", - "\n", - "\n", - "def expand_previous_messages(df):\n", - " \"\"\"\n", - " Expands the previous_messages column into separate columns.\n", - " \"\"\"\n", - " # First parse the string into actual list of dicts\n", - " df['previous_messages'] = df['previous_messages'].apply(json.loads)\n", - "\n", - " # Then create separate columns for each message\n", - " for i in range(MAX_NUM_PREVIOUS_MESSAGES):\n", - " df[f'previous_message_{i+1}'] = df['previous_messages'].apply(\n", - " lambda x: x[i] if i < len(x) else None\n", - " )\n", - "\n", - " # Drop the original previous_messages column if desired\n", - " return df.drop('previous_messages', axis=1)\n", - "\n", - "\n", - "\n", - "def make_messages_readable(df):\n", - " \"\"\"\n", - " Makes the messages more readable.\n", - " \"\"\"\n", - " for i in range(MAX_NUM_PREVIOUS_MESSAGES):\n", - " df[f'previous_message_{i+1}'] = df[f'previous_message_{i+1}'].apply(\n", - " lambda x: \"|\"*10 + f\" {x['role']} \" + \"|\"*10 + \"\\n\\n\" + f\"{x['content']}\" if x is not None else None\n", - " )\n", - "\n", - " df['message'] = df.apply(\n", - " lambda row: \"|\"*10 + f\" {row['message_role']} \" + \"|\"*10 + \"\\n\\n\" + f\"{row['message']}\" if row['message'] is not None else None,\n", - " axis=1\n", - " )\n", - " return df\n", - "\n", - "\n", - "def order_columns(df):\n", - " \"\"\"\n", - " Orders the columns in the way we want them.\n", - " \"\"\"\n", - " df = df[['question_id', 'question_type', 'multisession_index', 'session_index', 'message_index_within_session', 'message_index_across_sessions', 'session_date', 'message_role', 'num_previous_messages', 'message_has_answer', 'previous_message_1', 'previous_message_2', 'previous_message_3', 'previous_message_4', 'previous_message_5', 'message']]\n", - " return df\n", - "\n", - "\n", - "\n", - "def insert_answer_columns(df, num_prompt_instructions):\n", - "\n", - " for prompt_instruction_index in range(num_prompt_instructions, 0, -1):\n", - "\n", - " for i in range(MAX_NUM_PREVIOUS_MESSAGES, 0, -1):\n", - " # Insert after each previous message\n", - " column_tag = f\"({prompt_instruction_index}.{i})\"\n", - " df.insert(\n", - " loc=df.columns.get_loc(f'previous_message_{i}') + 1,\n", - " column=f'Answer to Prompt Instruction {prompt_instruction_index} {column_tag}',\n", - " value=''\n", - " )\n", - " df.insert(\n", - " loc=df.columns.get_loc(f'previous_message_{i}') + 2,\n", - " column=f'Done? {column_tag}',\n", - " value=''\n", - " )\n", - "\n", - " column_tag = f\"({prompt_instruction_index}.{MAX_NUM_PREVIOUS_MESSAGES+1})\"\n", - " # Insert after each previous message\n", - " df.insert(\n", - " loc=df.columns.get_loc(f'message') + 1,\n", - " column=f'Answer to Prompt Instruction {prompt_instruction_index} {column_tag}',\n", - " value=''\n", - " )\n", - " df.insert(\n", - " loc=df.columns.get_loc(f'message') + 2,\n", - " column=f'Done? {column_tag}',\n", - " value=''\n", - " )\n", - "\n", - "\n", - "def insert_default_answers_round1(df):\n", - " \"\"\"\n", - " Inserts default answers for the first round of prompt instructions.\n", - " \"\"\"\n", - " for i in range(MAX_NUM_PREVIOUS_MESSAGES, 0, -1):\n", - " column_tag = f\"(1.{i})\"\n", - " answer_col = f'Answer to Prompt Instruction 1 {column_tag}'\n", - " msg_col = f'previous_message_{i}'\n", - " \n", - " # Set default value based on role from previous message\n", - " df[answer_col] = df[msg_col].apply(\n", - " lambda x: f\"[${x['role']}$, ]\" if x is not None else \"\"\n", - " )\n", - "\n", - " # Handle the final message\n", - " column_tag = f\"(1.{MAX_NUM_PREVIOUS_MESSAGES+1})\"\n", - " answer_col = f'Answer to Prompt Instruction 1 {column_tag}'\n", - "\n", - " # Set default value based on role from current message\n", - " df[answer_col] = df.apply(\n", - " lambda row: f\"[${row['message_role']}$, ]\",\n", - " axis=1\n", - " )\n", - "\n", - "def insert_example_row(df, num_prompt_instructions):\n", - " \"\"\"\n", - " Inserts an example row at the top of the dataframe with 'EXAMPLE' as values.\n", - " \"\"\"\n", - " example_row = {col: \"EXAMPLE\" for col in df.columns}\n", - " # for i in range(2):\n", - " # for j in range(num_prompt_instructions):\n", - " # example_row[f\"Done? ({j+1}.{i+1})\"] = \"x\"\n", - " df.loc[-1] = example_row\n", - " df.index = df.index + 1\n", - " df.sort_index(inplace=True)\n", - " return df\n", - "\n", - "\n", - "\n", - "def transform_eval_minidataset(df):\n", - " \"\"\"\n", - " Transforms the eval mini dataset so that there is a row for every message in previous messages.\n", - " \"\"\"\n", - "\n", - " df = df.rename(columns={col: f\"snippet_{col}\" for col in df.columns})\n", - "\n", - " ### Add new columns\n", - " df.reset_index(drop=True, inplace=True)\n", - " df['snippet_index'] = df.index\n", - "\n", - " transformed_rows = []\n", - "\n", - " for _, row in df.iterrows():\n", - " previous_messages = json.loads(row['snippet_previous_messages'])\n", - " \n", - " for i, message in enumerate(previous_messages):\n", - " new_row = row.copy()\n", - " new_row['message_index_within_snippet'] = i\n", - " new_row['input_message'] = json.dumps(message)\n", - " new_row['input_previous_messages'] = json.dumps(previous_messages[:i])\n", - " transformed_rows.append(new_row)\n", - "\n", - " new_row = row.copy()\n", - " new_row['message_index_within_snippet'] = len(previous_messages)\n", - " new_row['input_message'] = row['snippet_message']\n", - " new_row['input_previous_messages'] = row['snippet_previous_messages']\n", - " transformed_rows.append(new_row)\n", - "\n", - " transformed_df = pd.DataFrame(transformed_rows)\n", - "\n", - " transformed_rows = []\n", - " task_names = [\"extract_nodes\", \"dedupe_nodes\", \"extract_edges\", \"dedupe_edges\", \"extract_edge_dates\", \"edge_invalidation\"]\n", - " for _, row in transformed_df.iterrows():\n", - " for task_index, task_name in enumerate(task_names):\n", - " new_row = row.copy()\n", - " new_row['task_name'] = task_name\n", - " new_row['task_index'] = task_index\n", - " transformed_rows.append(new_row)\n", - "\n", - " transformed_df = pd.DataFrame(transformed_rows)\n", - "\n", - " # Reorder columns\n", - " transformed_df = transformed_df[['snippet_index', 'message_index_within_snippet', 'task_index', 'task_name', 'snippet_message', 'snippet_previous_messages', 'input_message', 'input_previous_messages']]#, 'input_extracted_nodes', 'input_existing_relevant_nodes', 'input_extracted_edges', 'input_existing_relevant_edges', 'output_zep', 'output_gpt4o', 'output_human']]\n", - "\n", - " # Ensure to reset the indices to be sequential\n", - " transformed_df.reset_index(drop=True, inplace=True)\n", - "\n", - " return transformed_df\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "# Set pandas settings to display all columns and have max width of columns\n", - "pd.set_option('display.max_columns', 100)\n", - "pd.set_option('display.max_rows', 100)\n", - "pd.set_option('display.max_colwidth', 120)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
question_idquestion_typemultisession_indexsession_indexmessage_index_within_sessionmessage_index_across_sessionssession_datemessageprevious_messagesnum_previous_messagesmessage_has_answer
6686e47becbasingle-session-user2860552023-05-21 11:54:00{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...5True
\n", - "
" - ], - "text/plain": [ - " question_id question_type multisession_index session_index \\\n", - "6686 e47becba single-session-user 286 0 \n", - "\n", - " message_index_within_session message_index_across_sessions \\\n", - "6686 5 5 \n", - "\n", - " session_date \\\n", - "6686 2023-05-21 11:54:00 \n", - "\n", - " message \\\n", - "6686 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "\n", - " previous_messages \\\n", - "6686 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "\n", - " num_previous_messages message_has_answer \n", - "6686 5 True " - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "######## Filtering to only snippets/rows we want\n", - "lme_dataset_df_filtered = filter_for_zep_labelling(lme_dataset_df)\n", - "lme_dataset_df_filtered.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of rows: 36\n", - "Number of columns: 8\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
snippet_indexmessage_index_within_snippettask_indextask_namesnippet_messagesnippet_previous_messagesinput_messageinput_previous_messages
0000extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
1001dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
2002extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
3003dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
4004extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
5005edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
6010extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
7011dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
8012extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
9013dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
10014extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
11015edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
12020extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
13021dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
14022extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
15023dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
16024extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
17025edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
18030extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
19031dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
20032extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
21033dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
22034extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
23035edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
24040extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
25041dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
26042extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
27043dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
28044extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
29045edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
30050extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
31051dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
32052extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
33053dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
34054extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
35055edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
\n", - "
" - ], - "text/plain": [ - " snippet_index message_index_within_snippet task_index \\\n", - "0 0 0 0 \n", - "1 0 0 1 \n", - "2 0 0 2 \n", - "3 0 0 3 \n", - "4 0 0 4 \n", - "5 0 0 5 \n", - "6 0 1 0 \n", - "7 0 1 1 \n", - "8 0 1 2 \n", - "9 0 1 3 \n", - "10 0 1 4 \n", - "11 0 1 5 \n", - "12 0 2 0 \n", - "13 0 2 1 \n", - "14 0 2 2 \n", - "15 0 2 3 \n", - "16 0 2 4 \n", - "17 0 2 5 \n", - "18 0 3 0 \n", - "19 0 3 1 \n", - "20 0 3 2 \n", - "21 0 3 3 \n", - "22 0 3 4 \n", - "23 0 3 5 \n", - "24 0 4 0 \n", - "25 0 4 1 \n", - "26 0 4 2 \n", - "27 0 4 3 \n", - "28 0 4 4 \n", - "29 0 4 5 \n", - "30 0 5 0 \n", - "31 0 5 1 \n", - "32 0 5 2 \n", - "33 0 5 3 \n", - "34 0 5 4 \n", - "35 0 5 5 \n", - "\n", - " task_name \\\n", - "0 extract_nodes \n", - "1 dedupe_nodes \n", - "2 extract_edges \n", - "3 dedupe_edges \n", - "4 extract_edge_dates \n", - "5 edge_invalidation \n", - "6 extract_nodes \n", - "7 dedupe_nodes \n", - "8 extract_edges \n", - "9 dedupe_edges \n", - "10 extract_edge_dates \n", - "11 edge_invalidation \n", - "12 extract_nodes \n", - "13 dedupe_nodes \n", - "14 extract_edges \n", - "15 dedupe_edges \n", - "16 extract_edge_dates \n", - "17 edge_invalidation \n", - "18 extract_nodes \n", - "19 dedupe_nodes \n", - "20 extract_edges \n", - "21 dedupe_edges \n", - "22 extract_edge_dates \n", - "23 edge_invalidation \n", - "24 extract_nodes \n", - "25 dedupe_nodes \n", - "26 extract_edges \n", - "27 dedupe_edges \n", - "28 extract_edge_dates \n", - "29 edge_invalidation \n", - "30 extract_nodes \n", - "31 dedupe_nodes \n", - "32 extract_edges \n", - "33 dedupe_edges \n", - "34 extract_edge_dates \n", - "35 edge_invalidation \n", - "\n", - " snippet_message \\\n", - "0 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "1 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "2 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "3 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "4 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "5 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "6 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "7 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "8 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "9 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "10 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "11 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "12 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "13 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "14 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "15 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "16 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "17 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "18 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "19 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "20 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "21 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "22 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "23 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "24 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "25 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "26 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "27 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "28 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "29 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "30 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "31 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "32 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "33 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "34 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "35 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "\n", - " snippet_previous_messages \\\n", - "0 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "1 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "2 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "3 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "4 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "5 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "6 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "7 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "8 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "9 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "10 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "11 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "12 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "13 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "14 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "15 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "16 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "17 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "18 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "19 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "20 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "21 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "22 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "23 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "24 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "25 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "26 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "27 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "28 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "29 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "30 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "31 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "32 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "33 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "34 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "35 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "\n", - " input_message \\\n", - "0 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "1 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "2 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "3 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "4 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "5 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "6 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "7 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "8 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "9 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "10 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "11 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "12 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "13 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "14 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "15 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "16 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "17 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "18 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "19 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "20 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "21 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "22 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "23 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "24 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "25 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "26 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "27 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "28 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "29 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "30 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "31 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "32 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "33 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "34 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "35 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "\n", - " input_previous_messages \n", - "0 [] \n", - "1 [] \n", - "2 [] \n", - "3 [] \n", - "4 [] \n", - "5 [] \n", - "6 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "7 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "8 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "9 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "10 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "11 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "12 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "13 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "14 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "15 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "16 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "17 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "18 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "19 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "20 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "21 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "22 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "23 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "24 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "25 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "26 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "27 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "28 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "29 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "30 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "31 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "32 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "33 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "34 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "35 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... " - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#################### Create the eval CSV\n", - "eval_minidataset = lme_dataset_df_filtered.copy()\n", - "eval_minidataset = transform_eval_minidataset(eval_minidataset)\n", - "# Print the number of rows and columns\n", - "print(f\"Number of rows: {len(eval_minidataset)}\")\n", - "print(f\"Number of columns: {len(eval_minidataset.columns)}\")\n", - "eval_minidataset.head(100)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Number of rows: 36\n", - "Number of columns: 9\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
snippet_indexmessage_index_within_snippettask_indextask_namesnippet_messagesnippet_previous_messagesinput_messageinput_previous_messagesoutput_gpt4o_mini
0000extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[][{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
1001dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[][{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
2002extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[][{\"uuid\": \"cda66daf-afe0-4098-a2b6-c3f71f30fe85\", \"group_id\": \"\", \"source_node_uuid\": \"afb12271-73aa-4d41-a8a5-c5411...
3003dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]None
4004extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]None
5005edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]None
6010extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
7011dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
8012extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"3ad7f861-0470-4955-b698-cc67fe186bc5\", \"group_id\": \"\", \"source_node_uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d...
9013dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
10014extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
11015edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
12020extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"c65758f0-752c-418a-ad6b-2d8a74cdd4ce\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
13021dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
14022extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"165b9582-acee-46ed-93f9-216c17344642\", \"group_id\": \"\", \"source_node_uuid\": \"c65758f0-752c-418a-ad6b-2d8a7...
15023dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
16024extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
17025edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
18030extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"ef20491c-a031-4aea-8e51-0d0c47c98245\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
19031dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
20032extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"032a54c8-15b2-47b0-a89e-79f985938661\", \"group_id\": \"\", \"source_node_uuid\": \"aa977349-dd97-4af2-942b-02844...
21033dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
22034extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
23035edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
24040extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"06bc8194-ee62-46bd-9307-7e96001e4d07\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
25041dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
26042extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"eedb6466-4764-4c23-9107-c03b49979978\", \"group_id\": \"\", \"source_node_uuid\": \"06bc8194-ee62-46bd-9307-7e960...
27043dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
28044extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
29045edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
30050extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"e04cfd2d-c6e4-4372-8385-6a7c89dfe9b6\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
31051dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
32052extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"fd4b2107-3660-4eb6-9765-c89d709ac57f\", \"group_id\": \"\", \"source_node_uuid\": \"e04cfd2d-c6e4-4372-8385-6a7c8...
33053dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
34054extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
35055edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
\n", - "
" - ], - "text/plain": [ - " snippet_index message_index_within_snippet task_index \\\n", - "0 0 0 0 \n", - "1 0 0 1 \n", - "2 0 0 2 \n", - "3 0 0 3 \n", - "4 0 0 4 \n", - "5 0 0 5 \n", - "6 0 1 0 \n", - "7 0 1 1 \n", - "8 0 1 2 \n", - "9 0 1 3 \n", - "10 0 1 4 \n", - "11 0 1 5 \n", - "12 0 2 0 \n", - "13 0 2 1 \n", - "14 0 2 2 \n", - "15 0 2 3 \n", - "16 0 2 4 \n", - "17 0 2 5 \n", - "18 0 3 0 \n", - "19 0 3 1 \n", - "20 0 3 2 \n", - "21 0 3 3 \n", - "22 0 3 4 \n", - "23 0 3 5 \n", - "24 0 4 0 \n", - "25 0 4 1 \n", - "26 0 4 2 \n", - "27 0 4 3 \n", - "28 0 4 4 \n", - "29 0 4 5 \n", - "30 0 5 0 \n", - "31 0 5 1 \n", - "32 0 5 2 \n", - "33 0 5 3 \n", - "34 0 5 4 \n", - "35 0 5 5 \n", - "\n", - " task_name \\\n", - "0 extract_nodes \n", - "1 dedupe_nodes \n", - "2 extract_edges \n", - "3 dedupe_edges \n", - "4 extract_edge_dates \n", - "5 edge_invalidation \n", - "6 extract_nodes \n", - "7 dedupe_nodes \n", - "8 extract_edges \n", - "9 dedupe_edges \n", - "10 extract_edge_dates \n", - "11 edge_invalidation \n", - "12 extract_nodes \n", - "13 dedupe_nodes \n", - "14 extract_edges \n", - "15 dedupe_edges \n", - "16 extract_edge_dates \n", - "17 edge_invalidation \n", - "18 extract_nodes \n", - "19 dedupe_nodes \n", - "20 extract_edges \n", - "21 dedupe_edges \n", - "22 extract_edge_dates \n", - "23 edge_invalidation \n", - "24 extract_nodes \n", - "25 dedupe_nodes \n", - "26 extract_edges \n", - "27 dedupe_edges \n", - "28 extract_edge_dates \n", - "29 edge_invalidation \n", - "30 extract_nodes \n", - "31 dedupe_nodes \n", - "32 extract_edges \n", - "33 dedupe_edges \n", - "34 extract_edge_dates \n", - "35 edge_invalidation \n", - "\n", - " snippet_message \\\n", - "0 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "1 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "2 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "3 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "4 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "5 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "6 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "7 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "8 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "9 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "10 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "11 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "12 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "13 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "14 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "15 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "16 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "17 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "18 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "19 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "20 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "21 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "22 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "23 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "24 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "25 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "26 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "27 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "28 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "29 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "30 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "31 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "32 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "33 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "34 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "35 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "\n", - " snippet_previous_messages \\\n", - "0 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "1 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "2 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "3 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "4 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "5 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "6 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "7 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "8 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "9 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "10 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "11 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "12 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "13 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "14 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "15 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "16 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "17 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "18 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "19 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "20 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "21 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "22 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "23 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "24 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "25 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "26 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "27 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "28 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "29 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "30 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "31 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "32 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "33 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "34 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "35 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "\n", - " input_message \\\n", - "0 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "1 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "2 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "3 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "4 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "5 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", - "6 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "7 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "8 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "9 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "10 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "11 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", - "12 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "13 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "14 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "15 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "16 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "17 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", - "18 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "19 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "20 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "21 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "22 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "23 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", - "24 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "25 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "26 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "27 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "28 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "29 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", - "30 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "31 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "32 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "33 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "34 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "35 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", - "\n", - " input_previous_messages \\\n", - "0 [] \n", - "1 [] \n", - "2 [] \n", - "3 [] \n", - "4 [] \n", - "5 [] \n", - "6 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "7 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "8 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "9 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "10 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "11 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "12 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "13 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "14 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "15 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "16 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "17 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "18 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "19 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "20 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "21 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "22 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "23 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "24 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "25 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "26 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "27 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "28 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "29 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "30 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "31 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "32 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "33 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "34 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "35 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", - "\n", - " output_gpt4o_mini \n", - "0 [{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", - "1 [{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", - "2 [{\"uuid\": \"cda66daf-afe0-4098-a2b6-c3f71f30fe85\", \"group_id\": \"\", \"source_node_uuid\": \"afb12271-73aa-4d41-a8a5-c5411... \n", - "3 None \n", - "4 None \n", - "5 None \n", - "6 [{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", - "7 [{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", - "8 [{\"uuid\": \"3ad7f861-0470-4955-b698-cc67fe186bc5\", \"group_id\": \"\", \"source_node_uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d... \n", - "9 None \n", - "10 None \n", - "11 None \n", - "12 [{\"uuid\": \"c65758f0-752c-418a-ad6b-2d8a74cdd4ce\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", - "13 [{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", - "14 [{\"uuid\": \"165b9582-acee-46ed-93f9-216c17344642\", \"group_id\": \"\", \"source_node_uuid\": \"c65758f0-752c-418a-ad6b-2d8a7... \n", - "15 None \n", - "16 None \n", - "17 None \n", - "18 [{\"uuid\": \"ef20491c-a031-4aea-8e51-0d0c47c98245\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", - "19 [{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", - "20 [{\"uuid\": \"032a54c8-15b2-47b0-a89e-79f985938661\", \"group_id\": \"\", \"source_node_uuid\": \"aa977349-dd97-4af2-942b-02844... \n", - "21 None \n", - "22 None \n", - "23 None \n", - "24 [{\"uuid\": \"06bc8194-ee62-46bd-9307-7e96001e4d07\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", - "25 [{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", - "26 [{\"uuid\": \"eedb6466-4764-4c23-9107-c03b49979978\", \"group_id\": \"\", \"source_node_uuid\": \"06bc8194-ee62-46bd-9307-7e960... \n", - "27 None \n", - "28 None \n", - "29 None \n", - "30 [{\"uuid\": \"e04cfd2d-c6e4-4372-8385-6a7c89dfe9b6\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", - "31 [{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", - "32 [{\"uuid\": \"fd4b2107-3660-4eb6-9765-c89d709ac57f\", \"group_id\": \"\", \"source_node_uuid\": \"e04cfd2d-c6e4-4372-8385-6a7c8... \n", - "33 None \n", - "34 None \n", - "35 None " - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Insert gpt4o answers by doing ingestion in the right order and filling extra input columns as needed\n", - "model_name = 'gpt-4o-mini'\n", - "llm_config = LLMConfig(\n", - " api_key=os.getenv('OPENAI_API_KEY'),\n", - " model=model_name,\n", - ")\n", - "llm_client = OpenAIClient(config=llm_config)\n", - "output_column_name = 'output_gpt4o_mini'\n", - "eval_minidataset_labelled = await ingest_and_label_minidataset(llm_client, eval_minidataset, output_column_name)\n", - "\n", - "# Print the number of rows and columns\n", - "print(f\"Number of rows: {len(eval_minidataset_labelled)}\")\n", - "print(f\"Number of columns: {len(eval_minidataset_labelled.columns)}\")\n", - "eval_minidataset_labelled.head(100)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management apps that can help me prioritize my work and personal tasks? I've been using a planner, but I think I need something more digital.\"}\n", - "Fact: The user has been using a planner., Name: USES\n", - "----------------------------------------------------------------------------------------------------\n", - "\n", - "\n", - "Fact: The user is trying to organize their life better and is seeking task management apps., Name: SEEKS\n", - "----------------------------------------------------------------------------------------------------\n", - "\n", - "\n", - "Fact: The user aims to prioritize their work tasks., Name: AIMS_TO_PRIORITIZE\n", - "----------------------------------------------------------------------------------------------------\n", - "\n", - "\n", - "Fact: The user aims to prioritize their personal tasks., Name: AIMS_TO_PRIORITIZE\n", - "----------------------------------------------------------------------------------------------------\n", - "\n", - "\n", - "Fact: The user prefers a digital solution over a planner., Name: PREFERS\n", - "----------------------------------------------------------------------------------------------------\n", - "\n", - "\n" - ] - } - ], - "source": [ - "# print the input message column for the first row\n", - "index = 2\n", - "print(\"Input Message:\")\n", - "print(eval_minidataset_labelled.iloc[index]['input_message'])\n", - "print(\"-\"*100)\n", - "cell_value = eval_minidataset_labelled.iloc[index][output_column_name]\n", - "cell_value_dicts = json.loads(cell_value)\n", - "for dict in cell_value_dicts:\n", - " # Print only the 'fact' and 'name' values\n", - " print(f\"Fact: {dict.get('fact', 'N/A')}, Name: {dict.get('name', 'N/A')}\")\n", - " print(\"-\"*100)\n", - " print(\"\\n\")\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "#################### Create the human labelling CSV\n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "#################### Create the human labelling CSV (old)\n", - "\n", - "\n", - "# ######## Expanding the previous_messages column\n", - "# lme_dataset_df_filtered_human_labelling = expand_previous_messages(lme_dataset_df_filtered)\n", - "# lme_dataset_df_filtered_human_labelling.head()\n", - "\n", - "# ######## Order the columns in the way we want them\n", - "# lme_dataset_df_filtered_human_labelling = order_columns(lme_dataset_df_filtered_human_labelling)\n", - "# lme_dataset_df_filtered_human_labelling.head()\n", - "\n", - "# ######## Insert empty answer columns\n", - "# num_prompt_instructions = 1\n", - "# insert_answer_columns(lme_dataset_df_filtered_human_labelling, num_prompt_instructions)\n", - "# lme_dataset_df_filtered_human_labelling.head()\n", - "\n", - "# ######## Insert default values for the answers\n", - "# insert_default_answers_round1(lme_dataset_df_filtered_human_labelling)\n", - "# lme_dataset_df_filtered_human_labelling.head()\n", - "\n", - "# ######## Make the messages more readable\n", - "# lme_dataset_df_filtered_human_labelling = make_messages_readable(lme_dataset_df_filtered_human_labelling)\n", - "# lme_dataset_df_filtered_human_labelling.head(10)\n", - "\n", - "# ######## Add example row to the top\n", - "# insert_example_row(lme_dataset_df_filtered_human_labelling, num_prompt_instructions)\n", - "# lme_dataset_df_filtered_human_labelling.head(10)\n", - "\n", - "# ######## Save to csv\n", - "# lme_dataset_df_filtered_human_labelling.to_csv(\"lme_dataset_df_filtered_human_labelling.csv\", index=False)" - ] - } - ], - "metadata": { "colab": { - "provenance": [] + "base_uri": "https://localhost:8080/" }, - "kernelspec": { - "display_name": "Python (.venv)", - "language": "python", - "name": ".venv" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.0" + "id": "i9i9uUZ3pWQE", + "outputId": "84404bb8-5841-4f2f-dd87-7f909c6e95aa" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pandas in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (2.2.2)\r\n", + "Requirement already satisfied: graphiti-core in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (0.4.0)\r\n", + "Requirement already satisfied: numpy>=1.26.0 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from pandas) (2.1.3)\r\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from pandas) (2.9.0.post0)\r\n", + "Requirement already satisfied: pytz>=2020.1 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from pandas) (2024.2)\r\n", + "Requirement already satisfied: tzdata>=2022.7 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from pandas) (2024.1)\r\n", + "Requirement already satisfied: diskcache<6.0.0,>=5.6.3 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from graphiti-core) (5.6.3)\r\n", + "Requirement already satisfied: neo4j<6.0.0,>=5.23.0 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from graphiti-core) (5.26.0)\r\n", + "Requirement already satisfied: openai<2.0.0,>=1.53.0 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from graphiti-core) (1.54.3)\r\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.8.2 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from graphiti-core) (2.9.2)\r\n", + "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.1 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from graphiti-core) (1.0.1)\r\n", + "Requirement already satisfied: tenacity==9.0.0 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from graphiti-core) (9.0.0)\r\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (4.6.2.post1)\r\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (1.9.0)\r\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (0.27.2)\r\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (0.7.0)\r\n", + "Requirement already satisfied: sniffio in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (1.3.1)\r\n", + "Requirement already satisfied: tqdm>4 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (4.67.0)\r\n", + "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from openai<2.0.0,>=1.53.0->graphiti-core) (4.12.2)\r\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.8.2->graphiti-core) (0.7.0)\r\n", + "Requirement already satisfied: pydantic-core==2.23.4 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.8.2->graphiti-core) (2.23.4)\r\n", + "Requirement already satisfied: six>=1.5 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\r\n", + "Requirement already satisfied: idna>=2.8 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.53.0->graphiti-core) (3.10)\r\n", + "Requirement already satisfied: certifi in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.53.0->graphiti-core) (2024.8.30)\r\n", + "Requirement already satisfied: httpcore==1.* in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.53.0->graphiti-core) (1.0.6)\r\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /Users/prestonrasmussen/Library/Caches/pypoetry/virtualenvs/graphiti-core-XzHUgKi9-py3.12/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai<2.0.0,>=1.53.0->graphiti-core) (0.14.0)\r\n", + "\r\n", + "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m A new release of pip is available: \u001B[0m\u001B[31;49m24.1\u001B[0m\u001B[39;49m -> \u001B[0m\u001B[32;49m25.0.1\u001B[0m\r\n", + "\u001B[1m[\u001B[0m\u001B[34;49mnotice\u001B[0m\u001B[1;39;49m]\u001B[0m\u001B[39;49m To update, run: \u001B[0m\u001B[32;49mpip install --upgrade pip\u001B[0m\r\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] } + ], + "source": [ + "######## Installations - BE SURE TO MAKE YOUR OWN LOCAL VENV FIRST\n", + "\n", + "%pip install pandas graphiti-core" + ] }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-20T16:16:48.236627Z", + "start_time": "2025-03-20T16:16:47.667113Z" + }, + "id": "KgzveXyAp35v" + }, + "outputs": [], + "source": [ + "######## Imports\n", + "\n", + "import csv\n", + "import json\n", + "import os\n", + "from datetime import datetime\n", + "\n", + "import pandas as pd\n", + "from utils import dict_to_entity, entity_to_dict, ingest_and_label_minidataset\n", + "\n", + "from graphiti_core.llm_client import OpenAIClient\n", + "from graphiti_core.llm_client.config import LLMConfig\n", + "from graphiti_core.nodes import EntityNode" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2025-03-20T17:18:04.504351Z", + "start_time": "2025-03-20T17:18:04.080796Z" + }, + "id": "_RjEZnk5v530" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
question_idquestion_typemultisession_indexsession_indexmessage_index_within_sessionmessage_index_across_sessionssession_datemessageprevious_messagesnum_previous_messagesmessage_has_answer
0gpt4_2655b836temporal-reasoning00002023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I'm thinking of g...[]0False
1gpt4_2655b836temporal-reasoning00112023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"Choosing the...[{\"role\": \"user\", \"content\": \"I'm thinking of ...1False
2gpt4_2655b836temporal-reasoning00222023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I've been doing s...[{\"role\": \"user\", \"content\": \"I'm thinking of ...2True
3gpt4_2655b836temporal-reasoning00332023-04-10 14:47:00{\"role\": \"assistant\", \"content\": \"That's great...[{\"role\": \"user\", \"content\": \"I'm thinking of ...3False
4gpt4_2655b836temporal-reasoning00442023-04-10 14:47:00{\"role\": \"user\", \"content\": \"I'll definitely a...[{\"role\": \"user\", \"content\": \"I'm thinking of ...4False
\n", + "
" + ], + "text/plain": [ + " question_id question_type multisession_index session_index \\\n", + "0 gpt4_2655b836 temporal-reasoning 0 0 \n", + "1 gpt4_2655b836 temporal-reasoning 0 0 \n", + "2 gpt4_2655b836 temporal-reasoning 0 0 \n", + "3 gpt4_2655b836 temporal-reasoning 0 0 \n", + "4 gpt4_2655b836 temporal-reasoning 0 0 \n", + "\n", + " message_index_within_session message_index_across_sessions \\\n", + "0 0 0 \n", + "1 1 1 \n", + "2 2 2 \n", + "3 3 3 \n", + "4 4 4 \n", + "\n", + " session_date message \\\n", + "0 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'm thinking of g... \n", + "1 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Choosing the... \n", + "2 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I've been doing s... \n", + "3 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"That's great... \n", + "4 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'll definitely a... \n", + "\n", + " previous_messages num_previous_messages \\\n", + "0 [] 0 \n", + "1 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 1 \n", + "2 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 2 \n", + "3 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 3 \n", + "4 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 4 \n", + "\n", + " message_has_answer \n", + "0 False \n", + "1 False \n", + "2 True \n", + "3 False \n", + "4 False " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "######## Load the eval dataset\n", + "folder_path = './longmemeval_data/snippetized_data'\n", + "file_path = os.path.join(folder_path, 'longmemeval_oracle_snippetized.csv')\n", + "lme_dataset_df = pd.read_csv(file_path)\n", + "lme_dataset_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "######## Methods\n", + "\n", + "MAX_NUM_PREVIOUS_MESSAGES = 5\n", + "\n", + "\n", + "def filter_for_zep_labelling(df):\n", + " \"\"\"\n", + " Filters the dataset for the snippets we want to use for Zep labelling.\n", + " \"\"\"\n", + "\n", + " # Filter to only rows where question_type = single_session_user\n", + " df = df[df['question_type'] == 'single-session-user']\n", + "\n", + " # Filter only where message_has_answer = True\n", + " df = df[df['message_has_answer'] == True]\n", + "\n", + " # Filter to only rows where num_previous_messages = 5\n", + " df = df[df['num_previous_messages'] == 5]\n", + "\n", + " # Limit to only 5 rows\n", + " df = df.head(1)\n", + "\n", + " return df\n", + "\n", + "\n", + "def expand_previous_messages(df):\n", + " \"\"\"\n", + " Expands the previous_messages column into separate columns.\n", + " \"\"\"\n", + " # First parse the string into actual list of dicts\n", + " df['previous_messages'] = df['previous_messages'].apply(json.loads)\n", + "\n", + " # Then create separate columns for each message\n", + " for i in range(MAX_NUM_PREVIOUS_MESSAGES):\n", + " df[f'previous_message_{i + 1}'] = df['previous_messages'].apply(\n", + " lambda x: x[i] if i < len(x) else None\n", + " )\n", + "\n", + " # Drop the original previous_messages column if desired\n", + " return df.drop('previous_messages', axis=1)\n", + "\n", + "\n", + "def make_messages_readable(df):\n", + " \"\"\"\n", + " Makes the messages more readable.\n", + " \"\"\"\n", + " for i in range(MAX_NUM_PREVIOUS_MESSAGES):\n", + " df[f'previous_message_{i + 1}'] = df[f'previous_message_{i + 1}'].apply(\n", + " lambda x: '|' * 10 + f\" {x['role']} \" + '|' * 10 + '\\n\\n' + f\"{x['content']}\"\n", + " if x is not None\n", + " else None\n", + " )\n", + "\n", + " df['message'] = df.apply(\n", + " lambda row: '|' * 10\n", + " + f\" {row['message_role']} \"\n", + " + '|' * 10\n", + " + '\\n\\n'\n", + " + f\"{row['message']}\"\n", + " if row['message'] is not None\n", + " else None,\n", + " axis=1,\n", + " )\n", + " return df\n", + "\n", + "\n", + "def order_columns(df):\n", + " \"\"\"\n", + " Orders the columns in the way we want them.\n", + " \"\"\"\n", + " df = df[\n", + " [\n", + " 'question_id',\n", + " 'question_type',\n", + " 'multisession_index',\n", + " 'session_index',\n", + " 'message_index_within_session',\n", + " 'message_index_across_sessions',\n", + " 'session_date',\n", + " 'message_role',\n", + " 'num_previous_messages',\n", + " 'message_has_answer',\n", + " 'previous_message_1',\n", + " 'previous_message_2',\n", + " 'previous_message_3',\n", + " 'previous_message_4',\n", + " 'previous_message_5',\n", + " 'message',\n", + " ]\n", + " ]\n", + " return df\n", + "\n", + "\n", + "def insert_answer_columns(df, num_prompt_instructions):\n", + " for prompt_instruction_index in range(num_prompt_instructions, 0, -1):\n", + " for i in range(MAX_NUM_PREVIOUS_MESSAGES, 0, -1):\n", + " # Insert after each previous message\n", + " column_tag = f'({prompt_instruction_index}.{i})'\n", + " df.insert(\n", + " loc=df.columns.get_loc(f'previous_message_{i}') + 1,\n", + " column=f'Answer to Prompt Instruction {prompt_instruction_index} {column_tag}',\n", + " value='',\n", + " )\n", + " df.insert(\n", + " loc=df.columns.get_loc(f'previous_message_{i}') + 2,\n", + " column=f'Done? {column_tag}',\n", + " value='',\n", + " )\n", + "\n", + " column_tag = f'({prompt_instruction_index}.{MAX_NUM_PREVIOUS_MESSAGES + 1})'\n", + " # Insert after each previous message\n", + " df.insert(\n", + " loc=df.columns.get_loc(f'message') + 1,\n", + " column=f'Answer to Prompt Instruction {prompt_instruction_index} {column_tag}',\n", + " value='',\n", + " )\n", + " df.insert(\n", + " loc=df.columns.get_loc(f'message') + 2,\n", + " column=f'Done? {column_tag}',\n", + " value='',\n", + " )\n", + "\n", + "\n", + "def insert_default_answers_round1(df):\n", + " \"\"\"\n", + " Inserts default answers for the first round of prompt instructions.\n", + " \"\"\"\n", + " for i in range(MAX_NUM_PREVIOUS_MESSAGES, 0, -1):\n", + " column_tag = f'(1.{i})'\n", + " answer_col = f'Answer to Prompt Instruction 1 {column_tag}'\n", + " msg_col = f'previous_message_{i}'\n", + "\n", + " # Set default value based on role from previous message\n", + " df[answer_col] = df[msg_col].apply(lambda x: f\"[${x['role']}$, ]\" if x is not None else '')\n", + "\n", + " # Handle the final message\n", + " column_tag = f'(1.{MAX_NUM_PREVIOUS_MESSAGES + 1})'\n", + " answer_col = f'Answer to Prompt Instruction 1 {column_tag}'\n", + "\n", + " # Set default value based on role from current message\n", + " df[answer_col] = df.apply(lambda row: f\"[${row['message_role']}$, ]\", axis=1)\n", + "\n", + "\n", + "def insert_example_row(df, num_prompt_instructions):\n", + " \"\"\"\n", + " Inserts an example row at the top of the dataframe with 'EXAMPLE' as values.\n", + " \"\"\"\n", + " example_row = {col: 'EXAMPLE' for col in df.columns}\n", + " # for i in range(2):\n", + " # for j in range(num_prompt_instructions):\n", + " # example_row[f\"Done? ({j+1}.{i+1})\"] = \"x\"\n", + " df.loc[-1] = example_row\n", + " df.index = df.index + 1\n", + " df.sort_index(inplace=True)\n", + " return df\n", + "\n", + "\n", + "def transform_eval_minidataset(df):\n", + " \"\"\"\n", + " Transforms the eval mini dataset so that there is a row for every message in previous messages.\n", + " \"\"\"\n", + "\n", + " df = df.rename(columns={col: f'snippet_{col}' for col in df.columns})\n", + "\n", + " ### Add new columns\n", + " df.reset_index(drop=True, inplace=True)\n", + " df['snippet_index'] = df.index\n", + "\n", + " transformed_rows = []\n", + "\n", + " for _, row in df.iterrows():\n", + " previous_messages = json.loads(row['snippet_previous_messages'])\n", + "\n", + " for i, message in enumerate(previous_messages):\n", + " new_row = row.copy()\n", + " new_row['message_index_within_snippet'] = i\n", + " new_row['input_message'] = json.dumps(message)\n", + " new_row['input_previous_messages'] = json.dumps(previous_messages[:i])\n", + " transformed_rows.append(new_row)\n", + "\n", + " new_row = row.copy()\n", + " new_row['message_index_within_snippet'] = len(previous_messages)\n", + " new_row['input_message'] = row['snippet_message']\n", + " new_row['input_previous_messages'] = row['snippet_previous_messages']\n", + " transformed_rows.append(new_row)\n", + "\n", + " transformed_df = pd.DataFrame(transformed_rows)\n", + "\n", + " transformed_rows = []\n", + " task_names = [\n", + " 'extract_nodes',\n", + " 'dedupe_nodes',\n", + " 'extract_edges',\n", + " 'dedupe_edges',\n", + " 'extract_edge_dates',\n", + " 'edge_invalidation',\n", + " ]\n", + " for _, row in transformed_df.iterrows():\n", + " for task_index, task_name in enumerate(task_names):\n", + " new_row = row.copy()\n", + " new_row['task_name'] = task_name\n", + " new_row['task_index'] = task_index\n", + " transformed_rows.append(new_row)\n", + "\n", + " transformed_df = pd.DataFrame(transformed_rows)\n", + "\n", + " # Reorder columns\n", + " transformed_df = transformed_df[\n", + " [\n", + " 'snippet_index',\n", + " 'message_index_within_snippet',\n", + " 'task_index',\n", + " 'task_name',\n", + " 'snippet_message',\n", + " 'snippet_previous_messages',\n", + " 'input_message',\n", + " 'input_previous_messages',\n", + " ]\n", + " ] # , 'input_extracted_nodes', 'input_existing_relevant_nodes', 'input_extracted_edges', 'input_existing_relevant_edges', 'output_zep', 'output_gpt4o', 'output_human']]\n", + "\n", + " # Ensure to reset the indices to be sequential\n", + " transformed_df.reset_index(drop=True, inplace=True)\n", + "\n", + " return transformed_df" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# Set pandas settings to display all columns and have max width of columns\n", + "pd.set_option('display.max_columns', 100)\n", + "pd.set_option('display.max_rows', 100)\n", + "pd.set_option('display.max_colwidth', 120)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
question_idquestion_typemultisession_indexsession_indexmessage_index_within_sessionmessage_index_across_sessionssession_datemessageprevious_messagesnum_previous_messagesmessage_has_answer
6686e47becbasingle-session-user2860552023-05-21 11:54:00{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...5True
\n", + "
" + ], + "text/plain": [ + " question_id question_type multisession_index session_index \\\n", + "6686 e47becba single-session-user 286 0 \n", + "\n", + " message_index_within_session message_index_across_sessions \\\n", + "6686 5 5 \n", + "\n", + " session_date \\\n", + "6686 2023-05-21 11:54:00 \n", + "\n", + " message \\\n", + "6686 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "\n", + " previous_messages \\\n", + "6686 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "\n", + " num_previous_messages message_has_answer \n", + "6686 5 True " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "######## Filtering to only snippets/rows we want\n", + "lme_dataset_df_filtered = filter_for_zep_labelling(lme_dataset_df)\n", + "lme_dataset_df_filtered.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of rows: 36\n", + "Number of columns: 8\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
snippet_indexmessage_index_within_snippettask_indextask_namesnippet_messagesnippet_previous_messagesinput_messageinput_previous_messages
0000extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
1001dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
2002extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
3003dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
4004extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
5005edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]
6010extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
7011dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
8012extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
9013dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
10014extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
11015edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
12020extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
13021dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
14022extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
15023dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
16024extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
17025edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
18030extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
19031dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
20032extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
21033dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
22034extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
23035edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
24040extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
25041dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
26042extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
27043dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
28044extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
29045edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
30050extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
31051dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
32052extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
33053dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
34054extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
35055edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...
\n", + "
" + ], + "text/plain": [ + " snippet_index message_index_within_snippet task_index \\\n", + "0 0 0 0 \n", + "1 0 0 1 \n", + "2 0 0 2 \n", + "3 0 0 3 \n", + "4 0 0 4 \n", + "5 0 0 5 \n", + "6 0 1 0 \n", + "7 0 1 1 \n", + "8 0 1 2 \n", + "9 0 1 3 \n", + "10 0 1 4 \n", + "11 0 1 5 \n", + "12 0 2 0 \n", + "13 0 2 1 \n", + "14 0 2 2 \n", + "15 0 2 3 \n", + "16 0 2 4 \n", + "17 0 2 5 \n", + "18 0 3 0 \n", + "19 0 3 1 \n", + "20 0 3 2 \n", + "21 0 3 3 \n", + "22 0 3 4 \n", + "23 0 3 5 \n", + "24 0 4 0 \n", + "25 0 4 1 \n", + "26 0 4 2 \n", + "27 0 4 3 \n", + "28 0 4 4 \n", + "29 0 4 5 \n", + "30 0 5 0 \n", + "31 0 5 1 \n", + "32 0 5 2 \n", + "33 0 5 3 \n", + "34 0 5 4 \n", + "35 0 5 5 \n", + "\n", + " task_name \\\n", + "0 extract_nodes \n", + "1 dedupe_nodes \n", + "2 extract_edges \n", + "3 dedupe_edges \n", + "4 extract_edge_dates \n", + "5 edge_invalidation \n", + "6 extract_nodes \n", + "7 dedupe_nodes \n", + "8 extract_edges \n", + "9 dedupe_edges \n", + "10 extract_edge_dates \n", + "11 edge_invalidation \n", + "12 extract_nodes \n", + "13 dedupe_nodes \n", + "14 extract_edges \n", + "15 dedupe_edges \n", + "16 extract_edge_dates \n", + "17 edge_invalidation \n", + "18 extract_nodes \n", + "19 dedupe_nodes \n", + "20 extract_edges \n", + "21 dedupe_edges \n", + "22 extract_edge_dates \n", + "23 edge_invalidation \n", + "24 extract_nodes \n", + "25 dedupe_nodes \n", + "26 extract_edges \n", + "27 dedupe_edges \n", + "28 extract_edge_dates \n", + "29 edge_invalidation \n", + "30 extract_nodes \n", + "31 dedupe_nodes \n", + "32 extract_edges \n", + "33 dedupe_edges \n", + "34 extract_edge_dates \n", + "35 edge_invalidation \n", + "\n", + " snippet_message \\\n", + "0 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "1 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "2 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "3 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "4 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "5 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "6 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "7 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "8 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "9 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "10 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "11 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "12 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "13 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "14 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "15 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "16 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "17 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "18 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "19 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "20 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "21 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "22 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "23 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "24 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "25 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "26 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "27 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "28 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "29 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "30 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "31 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "32 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "33 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "34 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "35 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "\n", + " snippet_previous_messages \\\n", + "0 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "1 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "2 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "3 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "4 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "5 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "6 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "7 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "8 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "9 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "10 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "11 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "12 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "13 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "14 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "15 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "16 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "17 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "18 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "19 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "20 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "21 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "22 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "23 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "24 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "25 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "26 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "27 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "28 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "29 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "30 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "31 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "32 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "33 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "34 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "35 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "\n", + " input_message \\\n", + "0 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "1 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "2 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "3 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "4 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "5 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "6 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "7 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "8 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "9 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "10 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "11 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "12 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "13 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "14 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "15 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "16 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "17 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "18 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "19 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "20 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "21 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "22 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "23 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "24 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "25 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "26 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "27 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "28 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "29 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "30 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "31 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "32 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "33 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "34 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "35 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "\n", + " input_previous_messages \n", + "0 [] \n", + "1 [] \n", + "2 [] \n", + "3 [] \n", + "4 [] \n", + "5 [] \n", + "6 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "7 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "8 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "9 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "10 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "11 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "12 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "13 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "14 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "15 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "16 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "17 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "18 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "19 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "20 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "21 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "22 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "23 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "24 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "25 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "26 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "27 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "28 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "29 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "30 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "31 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "32 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "33 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "34 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "35 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#################### Create the eval CSV\n", + "eval_minidataset = lme_dataset_df_filtered.copy()\n", + "eval_minidataset = transform_eval_minidataset(eval_minidataset)\n", + "# Print the number of rows and columns\n", + "print(f'Number of rows: {len(eval_minidataset)}')\n", + "print(f'Number of columns: {len(eval_minidataset.columns)}')\n", + "eval_minidataset.head(100)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of rows: 36\n", + "Number of columns: 9\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
snippet_indexmessage_index_within_snippettask_indextask_namesnippet_messagesnippet_previous_messagesinput_messageinput_previous_messagesoutput_gpt4o_mini
0000extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[][{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
1001dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[][{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
2002extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[][{\"uuid\": \"cda66daf-afe0-4098-a2b6-c3f71f30fe85\", \"group_id\": \"\", \"source_node_uuid\": \"afb12271-73aa-4d41-a8a5-c5411...
3003dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]None
4004extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]None
5005edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app...[]None
6010extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
7011dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
8012extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"3ad7f861-0470-4955-b698-cc67fe186bc5\", \"group_id\": \"\", \"source_node_uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d...
9013dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
10014extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
11015edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
12020extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"c65758f0-752c-418a-ad6b-2d8a74cdd4ce\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
13021dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
14022extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"165b9582-acee-46ed-93f9-216c17344642\", \"group_id\": \"\", \"source_node_uuid\": \"c65758f0-752c-418a-ad6b-2d8a7...
15023dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
16024extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
17025edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
18030extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"ef20491c-a031-4aea-8e51-0d0c47c98245\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
19031dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
20032extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"032a54c8-15b2-47b0-a89e-79f985938661\", \"group_id\": \"\", \"source_node_uuid\": \"aa977349-dd97-4af2-942b-02844...
21033dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
22034extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
23035edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
24040extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"06bc8194-ee62-46bd-9307-7e96001e4d07\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
25041dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"...
26042extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"eedb6466-4764-4c23-9107-c03b49979978\", \"group_id\": \"\", \"source_node_uuid\": \"06bc8194-ee62-46bd-9307-7e960...
27043dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
28044extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
29045edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
30050extract_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"e04cfd2d-c6e4-4372-8385-6a7c89dfe9b6\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
31051dedupe_nodes{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create...
32052extract_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...[{\"uuid\": \"fd4b2107-3660-4eb6-9765-c89d709ac57f\", \"group_id\": \"\", \"source_node_uuid\": \"e04cfd2d-c6e4-4372-8385-6a7c8...
33053dedupe_edges{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
34054extract_edge_dates{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
35055edge_invalidation{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...{\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati...[{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap...None
\n", + "
" + ], + "text/plain": [ + " snippet_index message_index_within_snippet task_index \\\n", + "0 0 0 0 \n", + "1 0 0 1 \n", + "2 0 0 2 \n", + "3 0 0 3 \n", + "4 0 0 4 \n", + "5 0 0 5 \n", + "6 0 1 0 \n", + "7 0 1 1 \n", + "8 0 1 2 \n", + "9 0 1 3 \n", + "10 0 1 4 \n", + "11 0 1 5 \n", + "12 0 2 0 \n", + "13 0 2 1 \n", + "14 0 2 2 \n", + "15 0 2 3 \n", + "16 0 2 4 \n", + "17 0 2 5 \n", + "18 0 3 0 \n", + "19 0 3 1 \n", + "20 0 3 2 \n", + "21 0 3 3 \n", + "22 0 3 4 \n", + "23 0 3 5 \n", + "24 0 4 0 \n", + "25 0 4 1 \n", + "26 0 4 2 \n", + "27 0 4 3 \n", + "28 0 4 4 \n", + "29 0 4 5 \n", + "30 0 5 0 \n", + "31 0 5 1 \n", + "32 0 5 2 \n", + "33 0 5 3 \n", + "34 0 5 4 \n", + "35 0 5 5 \n", + "\n", + " task_name \\\n", + "0 extract_nodes \n", + "1 dedupe_nodes \n", + "2 extract_edges \n", + "3 dedupe_edges \n", + "4 extract_edge_dates \n", + "5 edge_invalidation \n", + "6 extract_nodes \n", + "7 dedupe_nodes \n", + "8 extract_edges \n", + "9 dedupe_edges \n", + "10 extract_edge_dates \n", + "11 edge_invalidation \n", + "12 extract_nodes \n", + "13 dedupe_nodes \n", + "14 extract_edges \n", + "15 dedupe_edges \n", + "16 extract_edge_dates \n", + "17 edge_invalidation \n", + "18 extract_nodes \n", + "19 dedupe_nodes \n", + "20 extract_edges \n", + "21 dedupe_edges \n", + "22 extract_edge_dates \n", + "23 edge_invalidation \n", + "24 extract_nodes \n", + "25 dedupe_nodes \n", + "26 extract_edges \n", + "27 dedupe_edges \n", + "28 extract_edge_dates \n", + "29 edge_invalidation \n", + "30 extract_nodes \n", + "31 dedupe_nodes \n", + "32 extract_edges \n", + "33 dedupe_edges \n", + "34 extract_edge_dates \n", + "35 edge_invalidation \n", + "\n", + " snippet_message \\\n", + "0 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "1 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "2 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "3 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "4 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "5 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "6 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "7 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "8 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "9 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "10 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "11 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "12 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "13 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "14 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "15 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "16 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "17 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "18 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "19 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "20 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "21 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "22 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "23 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "24 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "25 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "26 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "27 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "28 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "29 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "30 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "31 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "32 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "33 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "34 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "35 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "\n", + " snippet_previous_messages \\\n", + "0 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "1 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "2 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "3 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "4 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "5 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "6 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "7 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "8 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "9 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "10 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "11 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "12 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "13 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "14 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "15 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "16 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "17 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "18 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "19 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "20 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "21 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "22 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "23 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "24 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "25 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "26 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "27 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "28 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "29 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "30 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "31 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "32 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "33 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "34 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "35 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "\n", + " input_message \\\n", + "0 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "1 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "2 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "3 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "4 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "5 {\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management app... \n", + "6 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "7 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "8 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "9 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "10 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "11 {\"role\": \"assistant\", \"content\": \"Making the leap from a planner to a digital task management system! Congratulation... \n", + "12 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "13 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "14 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "15 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "16 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "17 {\"role\": \"user\", \"content\": \"I think I'll try out Todoist and Trello. I've heard a lot of good things about them. By... \n", + "18 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "19 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "20 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "21 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "22 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "23 {\"role\": \"assistant\", \"content\": \"Todoist and Trello are both excellent choices for task management.\\n\\nNow, about c... \n", + "24 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "25 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "26 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "27 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "28 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "29 {\"role\": \"user\", \"content\": \"I graduated with a degree in Business Administration, which has definitely helped me in... \n", + "30 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "31 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "32 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "33 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "34 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "35 {\"role\": \"assistant\", \"content\": \"Congratulations on your degree in Business Administration! That's a great foundati... \n", + "\n", + " input_previous_messages \\\n", + "0 [] \n", + "1 [] \n", + "2 [] \n", + "3 [] \n", + "4 [] \n", + "5 [] \n", + "6 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "7 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "8 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "9 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "10 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "11 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "12 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "13 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "14 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "15 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "16 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "17 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "18 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "19 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "20 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "21 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "22 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "23 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "24 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "25 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "26 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "27 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "28 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "29 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "30 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "31 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "32 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "33 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "34 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "35 [{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management ap... \n", + "\n", + " output_gpt4o_mini \n", + "0 [{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", + "1 [{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", + "2 [{\"uuid\": \"cda66daf-afe0-4098-a2b6-c3f71f30fe85\", \"group_id\": \"\", \"source_node_uuid\": \"afb12271-73aa-4d41-a8a5-c5411... \n", + "3 None \n", + "4 None \n", + "5 None \n", + "6 [{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", + "7 [{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", + "8 [{\"uuid\": \"3ad7f861-0470-4955-b698-cc67fe186bc5\", \"group_id\": \"\", \"source_node_uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d... \n", + "9 None \n", + "10 None \n", + "11 None \n", + "12 [{\"uuid\": \"c65758f0-752c-418a-ad6b-2d8a74cdd4ce\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", + "13 [{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", + "14 [{\"uuid\": \"165b9582-acee-46ed-93f9-216c17344642\", \"group_id\": \"\", \"source_node_uuid\": \"c65758f0-752c-418a-ad6b-2d8a7... \n", + "15 None \n", + "16 None \n", + "17 None \n", + "18 [{\"uuid\": \"ef20491c-a031-4aea-8e51-0d0c47c98245\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", + "19 [{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", + "20 [{\"uuid\": \"032a54c8-15b2-47b0-a89e-79f985938661\", \"group_id\": \"\", \"source_node_uuid\": \"aa977349-dd97-4af2-942b-02844... \n", + "21 None \n", + "22 None \n", + "23 None \n", + "24 [{\"uuid\": \"06bc8194-ee62-46bd-9307-7e96001e4d07\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", + "25 [{\"uuid\": \"afb12271-73aa-4d41-a8a5-c541169b85cf\", \"name\": \"user\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"created_at\"... \n", + "26 [{\"uuid\": \"eedb6466-4764-4c23-9107-c03b49979978\", \"group_id\": \"\", \"source_node_uuid\": \"06bc8194-ee62-46bd-9307-7e960... \n", + "27 None \n", + "28 None \n", + "29 None \n", + "30 [{\"uuid\": \"e04cfd2d-c6e4-4372-8385-6a7c89dfe9b6\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", + "31 [{\"uuid\": \"b68c71ec-87d6-43d8-9ec1-0155d8415400\", \"name\": \"assistant\", \"group_id\": \"\", \"labels\": [\"Entity\"], \"create... \n", + "32 [{\"uuid\": \"fd4b2107-3660-4eb6-9765-c89d709ac57f\", \"group_id\": \"\", \"source_node_uuid\": \"e04cfd2d-c6e4-4372-8385-6a7c8... \n", + "33 None \n", + "34 None \n", + "35 None " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Insert gpt4o answers by doing ingestion in the right order and filling extra input columns as needed\n", + "model_name = 'gpt-4o-mini'\n", + "llm_config = LLMConfig(\n", + " api_key=os.getenv('OPENAI_API_KEY'),\n", + " model=model_name,\n", + ")\n", + "llm_client = OpenAIClient(config=llm_config)\n", + "output_column_name = 'output_gpt4o_mini'\n", + "eval_minidataset_labelled = await ingest_and_label_minidataset(\n", + " llm_client, eval_minidataset, output_column_name\n", + ")\n", + "\n", + "# Print the number of rows and columns\n", + "print(f'Number of rows: {len(eval_minidataset_labelled)}')\n", + "print(f'Number of columns: {len(eval_minidataset_labelled.columns)}')\n", + "eval_minidataset_labelled.head(100)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"role\": \"user\", \"content\": \"I'm trying to organize my life a bit better, can you recommend some task management apps that can help me prioritize my work and personal tasks? I've been using a planner, but I think I need something more digital.\"}\n", + "Fact: The user has been using a planner., Name: USES\n", + "----------------------------------------------------------------------------------------------------\n", + "\n", + "\n", + "Fact: The user is trying to organize their life better and is seeking task management apps., Name: SEEKS\n", + "----------------------------------------------------------------------------------------------------\n", + "\n", + "\n", + "Fact: The user aims to prioritize their work tasks., Name: AIMS_TO_PRIORITIZE\n", + "----------------------------------------------------------------------------------------------------\n", + "\n", + "\n", + "Fact: The user aims to prioritize their personal tasks., Name: AIMS_TO_PRIORITIZE\n", + "----------------------------------------------------------------------------------------------------\n", + "\n", + "\n", + "Fact: The user prefers a digital solution over a planner., Name: PREFERS\n", + "----------------------------------------------------------------------------------------------------\n", + "\n", + "\n" + ] + } + ], + "source": [ + "# print the input message column for the first row\n", + "index = 2\n", + "print('Input Message:')\n", + "print(eval_minidataset_labelled.iloc[index]['input_message'])\n", + "print('-' * 100)\n", + "cell_value = eval_minidataset_labelled.iloc[index][output_column_name]\n", + "cell_value_dicts = json.loads(cell_value)\n", + "for dict in cell_value_dicts:\n", + " # Print only the 'fact' and 'name' values\n", + " print(f\"Fact: {dict.get('fact', 'N/A')}, Name: {dict.get('name', 'N/A')}\")\n", + " print('-' * 100)\n", + " print('\\n')" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "#################### Create the human labelling CSV\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "#################### Create the human labelling CSV (old)\n", + "\n", + "\n", + "# ######## Expanding the previous_messages column\n", + "# lme_dataset_df_filtered_human_labelling = expand_previous_messages(lme_dataset_df_filtered)\n", + "# lme_dataset_df_filtered_human_labelling.head()\n", + "\n", + "# ######## Order the columns in the way we want them\n", + "# lme_dataset_df_filtered_human_labelling = order_columns(lme_dataset_df_filtered_human_labelling)\n", + "# lme_dataset_df_filtered_human_labelling.head()\n", + "\n", + "# ######## Insert empty answer columns\n", + "# num_prompt_instructions = 1\n", + "# insert_answer_columns(lme_dataset_df_filtered_human_labelling, num_prompt_instructions)\n", + "# lme_dataset_df_filtered_human_labelling.head()\n", + "\n", + "# ######## Insert default values for the answers\n", + "# insert_default_answers_round1(lme_dataset_df_filtered_human_labelling)\n", + "# lme_dataset_df_filtered_human_labelling.head()\n", + "\n", + "# ######## Make the messages more readable\n", + "# lme_dataset_df_filtered_human_labelling = make_messages_readable(lme_dataset_df_filtered_human_labelling)\n", + "# lme_dataset_df_filtered_human_labelling.head(10)\n", + "\n", + "# ######## Add example row to the top\n", + "# insert_example_row(lme_dataset_df_filtered_human_labelling, num_prompt_instructions)\n", + "# lme_dataset_df_filtered_human_labelling.head(10)\n", + "\n", + "# ######## Save to csv\n", + "# lme_dataset_df_filtered_human_labelling.to_csv(\"lme_dataset_df_filtered_human_labelling.csv\", index=False)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/tests/evals/data/utils.py b/tests/evals/data/utils.py index 8ff855c4..b6382993 100644 --- a/tests/evals/data/utils.py +++ b/tests/evals/data/utils.py @@ -1,7 +1,10 @@ -import pandas as pd -from datetime import datetime, timedelta import json -from graphiti_core.nodes import EpisodicNode, EpisodeType, EntityNode +from datetime import datetime, timedelta + +import pandas as pd + +from graphiti_core.nodes import EntityNode, EpisodeType, EpisodicNode + def create_episodes_from_messages(input_message, input_previous_messages): """ @@ -11,28 +14,31 @@ def create_episodes_from_messages(input_message, input_previous_messages): current_time = datetime.now() # Create the current episode - role = input_message["role"] - content = input_message["content"] - message_content = f"{role}: {content}" + role = input_message['role'] + content = input_message['content'] + message_content = f'{role}: {content}' episode = EpisodicNode( - name="", - group_id="", + name='', + group_id='', source=EpisodeType.message, type=EpisodeType.message, - source_description="", + source_description='', content=message_content, valid_at=current_time, ) # Create previous episodes num_previous_messages = len(input_previous_messages) - previous_times = [current_time - timedelta(minutes=num_previous_messages - i) for i in range(num_previous_messages)] + previous_times = [ + current_time - timedelta(minutes=num_previous_messages - i) + for i in range(num_previous_messages) + ] previous_episodes = [ EpisodicNode( - name="", - group_id="", + name='', + group_id='', source=EpisodeType.message, - source_description="", + source_description='', content=f"{message['role']}: {message['content']}", valid_at=previous_time, ) @@ -41,10 +47,14 @@ def create_episodes_from_messages(input_message, input_previous_messages): return episode, previous_episodes + async def ingest_and_label_snippet(llm_client, snippet_df, output_column_name): # Import necessary functions - from graphiti_core.utils.maintenance.node_operations import extract_nodes, resolve_extracted_nodes from graphiti_core.utils.maintenance.edge_operations import extract_edges + from graphiti_core.utils.maintenance.node_operations import ( + extract_nodes, + resolve_extracted_nodes, + ) # Loop through each unique message_index_within_snippet in sorted order for message_index in sorted(snippet_df['message_index_within_snippet'].unique()): @@ -52,16 +62,24 @@ async def ingest_and_label_snippet(llm_client, snippet_df, output_column_name): #### Process 'extract_nodes' task extract_nodes_row = message_df[message_df['task_name'] == 'extract_nodes'] - assert len(extract_nodes_row) == 1, f"There should be exactly one row for 'extract_nodes' but there are {len(extract_nodes_row)}" + assert ( + len(extract_nodes_row) == 1 + ), f"There should be exactly one row for 'extract_nodes' but there are {len(extract_nodes_row)}" input_message = json.loads(extract_nodes_row.iloc[0]['input_message']) input_previous_messages = json.loads(extract_nodes_row.iloc[0]['input_previous_messages']) - episode, previous_episodes = create_episodes_from_messages(input_message, input_previous_messages) + episode, previous_episodes = create_episodes_from_messages( + input_message, input_previous_messages + ) extracted_nodes = await extract_nodes(llm_client, episode, previous_episodes) - snippet_df.at[extract_nodes_row.index[0], output_column_name] = json.dumps([entity_to_dict(node) for node in extracted_nodes]) - + snippet_df.at[extract_nodes_row.index[0], output_column_name] = json.dumps( + [entity_to_dict(node) for node in extracted_nodes] + ) + #### Process 'dedupe_nodes' task dedupe_nodes_row = message_df[message_df['task_name'] == 'dedupe_nodes'] - assert len(dedupe_nodes_row) == 1, "There should be exactly one row for 'dedupe_nodes' but there are {len(dedupe_nodes_row)}" + assert ( + len(dedupe_nodes_row) == 1 + ), "There should be exactly one row for 'dedupe_nodes' but there are {len(dedupe_nodes_row)}" # Calculate existing nodes list existing_nodes = [] @@ -71,8 +89,8 @@ async def ingest_and_label_snippet(llm_client, snippet_df, output_column_name): # Filter for previous messages with 'extract_nodes' task prev_message_df = snippet_df[ - (snippet_df['message_index_within_snippet'] == prev_message_index) & - (snippet_df['task_name'] == 'extract_nodes') + (snippet_df['message_index_within_snippet'] == prev_message_index) + & (snippet_df['task_name'] == 'extract_nodes') ] # Retrieve and deserialize the nodes @@ -82,12 +100,18 @@ async def ingest_and_label_snippet(llm_client, snippet_df, output_column_name): existing_nodes.extend(nodes) existing_nodes_lists = [existing_nodes for _ in range(len(extracted_nodes))] - resolved_nodes, uuid_map = await resolve_extracted_nodes(llm_client, extracted_nodes, existing_nodes_lists, episode, previous_episodes) - snippet_df.at[dedupe_nodes_row.index[0], output_column_name] = json.dumps([entity_to_dict(node) for node in resolved_nodes]) + resolved_nodes, uuid_map = await resolve_extracted_nodes( + llm_client, extracted_nodes, existing_nodes_lists, episode, previous_episodes + ) + snippet_df.at[dedupe_nodes_row.index[0], output_column_name] = json.dumps( + [entity_to_dict(node) for node in resolved_nodes] + ) #### Process 'extract_edges' task extract_edges_row = message_df[message_df['task_name'] == 'extract_edges'] - assert len(extract_edges_row) == 1, f"There should be exactly one row for 'extract_edges' but there are {len(extract_edges_row)}" + assert ( + len(extract_edges_row) == 1 + ), f"There should be exactly one row for 'extract_edges' but there are {len(extract_edges_row)}" extracted_edges = await extract_edges( llm_client, episode, @@ -95,7 +119,9 @@ async def ingest_and_label_snippet(llm_client, snippet_df, output_column_name): previous_episodes, group_id='', ) - snippet_df.at[extract_edges_row.index[0], output_column_name] = json.dumps([entity_to_dict(edge) for edge in extracted_edges]) + snippet_df.at[extract_edges_row.index[0], output_column_name] = json.dumps( + [entity_to_dict(edge) for edge in extracted_edges] + ) ########## TODO: Complete the implementation of the below @@ -131,17 +157,20 @@ async def ingest_and_label_minidataset(llm_client, minidataset_df, output_column minidataset_labelled_df = None for snippet_index in sorted(minidataset_df['snippet_index'].unique()): snippet_df = minidataset_df[minidataset_df['snippet_index'] == snippet_index] - + # Pass the output column name to the ingest_and_label_snippet function - snippet_df_labelled = await ingest_and_label_snippet(llm_client, snippet_df, output_column_name) - + snippet_df_labelled = await ingest_and_label_snippet( + llm_client, snippet_df, output_column_name + ) + if minidataset_labelled_df is None: minidataset_labelled_df = snippet_df_labelled else: minidataset_labelled_df = pd.concat([minidataset_labelled_df, snippet_df_labelled]) - + return minidataset_labelled_df + def entity_to_dict(entity): """ Convert an entity object to a dictionary, handling datetime serialization. @@ -152,6 +181,7 @@ def entity_to_dict(entity): entity_dict[key] = value.isoformat() # Convert datetime to ISO 8601 string return entity_dict + def dict_to_entity(entity_dict, entity_class): """ Convert a dictionary back to an entity object, handling datetime deserialization. @@ -163,4 +193,4 @@ def dict_to_entity(entity_dict, entity_class): except (ValueError, TypeError): # If parsing fails, keep the original value pass - return entity_class(**entity_dict) \ No newline at end of file + return entity_class(**entity_dict) diff --git a/tests/evals/eval_extract_nodes.py b/tests/evals/eval_extract_nodes.py index 720fb780..51446764 100644 --- a/tests/evals/eval_extract_nodes.py +++ b/tests/evals/eval_extract_nodes.py @@ -14,9 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. """ -import os +import csv # Add this import at the top of the file import json -from tests.evals.utils import setup_logging, ingest_snippet +import os from datetime import datetime, timedelta import pytest @@ -24,17 +24,11 @@ from dotenv import load_dotenv from graphiti_core.edges import EntityEdge, EpisodicEdge from graphiti_core.graphiti import Graphiti -from graphiti_core.nodes import EntityNode, EpisodicNode - -from graphiti_core.utils.maintenance.node_operations import extract_nodes from graphiti_core.llm_client import OpenAIClient from graphiti_core.llm_client.config import LLMConfig -from graphiti_core.nodes import EpisodeType - -import csv # Add this import at the top of the file - - - +from graphiti_core.nodes import EntityNode, EpisodeType, EpisodicNode +from graphiti_core.utils.maintenance.node_operations import extract_nodes +from tests.evals.utils import ingest_snippet, setup_logging ############# EVERYTHING BELOW IS OUTDATED @@ -63,68 +57,67 @@ async def general_extract_nodes_test(llm_client, data_sample): return hypothesis_node_names - - - def prepare_data_from_csv(data_file_name, question_id, session_idx, message_idx): - - samples_csv_path = "tests/evals/data/" + data_file_name + ".csv" + samples_csv_path = 'tests/evals/data/' + data_file_name + '.csv' # From CSV path, load everything with open(samples_csv_path, 'r') as file: csv_reader = csv.DictReader(file) lme_samples = list(csv_reader) - data_samples = [] # Loop through each row for row in lme_samples: - ### Prepare episode current_time = datetime.now() - message = json.loads(row["message"]) - role = message["role"] - content = message["content"] - message_content = role + ": " + content + message = json.loads(row['message']) + role = message['role'] + content = message['content'] + message_content = role + ': ' + content episode = EpisodicNode( - name="", - group_id="", + name='', + group_id='', source=EpisodeType.message, type=EpisodeType.message, - source_description="", + source_description='', content=message_content, - valid_at=current_time, + valid_at=current_time, ) ### Prepare previous episodes - previous_messages = json.loads(row["previous_messages"]) + previous_messages = json.loads(row['previous_messages']) num_previous_messages = len(previous_messages) - previous_times = [current_time - timedelta(minutes=num_previous_messages-i) for i in range(num_previous_messages)] - previous_episodes = [EpisodicNode( - name="", - group_id="", - source=EpisodeType.message, - source_description="", - content=message["role"] + ": " + message["content"], - valid_at=previous_time, - ) for message, previous_time in zip(previous_messages, previous_times)] + previous_times = [ + current_time - timedelta(minutes=num_previous_messages - i) + for i in range(num_previous_messages) + ] + previous_episodes = [ + EpisodicNode( + name='', + group_id='', + source=EpisodeType.message, + source_description='', + content=message['role'] + ': ' + message['content'], + valid_at=previous_time, + ) + for message, previous_time in zip(previous_messages, previous_times) + ] ### TODO: Prepare gold answer names ### Add to data samples list - data_samples.append({ - "episode": episode, - "previous_episodes": previous_episodes, - "gold_answer_names": [], - }) + data_samples.append( + { + 'episode': episode, + 'previous_episodes': previous_episodes, + 'gold_answer_names': [], + } + ) return data_samples - - - @pytest.mark.asyncio async def test_extract_nodes(): model_name = 'gpt-4o-mini' @@ -135,17 +128,16 @@ async def test_extract_nodes(): llm_client = OpenAIClient(config=llm_config) data_file_name = 'output_short' - question_id = "gpt4_2655b836" + question_id = 'gpt4_2655b836' session_idx = 0 message_idx = 0 data_samples = prepare_data_from_csv(data_file_name, question_id, session_idx, message_idx) for data_sample in data_samples: print(f"\n\nEpisode: {data_sample['episode']}") - print("*"*50) + print('*' * 50) print(f"Previous Episodes: {data_sample['previous_episodes']}") - print("*"*50) + print('*' * 50) # print(f"Gold Answer Names: {gold_answer_names}") await general_extract_nodes_test(llm_client, data_sample) -