1174 lines
58 KiB
Text
1174 lines
58 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"id": "i9i9uUZ3pWQE",
|
|
"outputId": "84404bb8-5841-4f2f-dd87-7f909c6e95aa"
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Collecting gdown\n",
|
|
" Using cached gdown-5.2.0-py3-none-any.whl.metadata (5.8 kB)\n",
|
|
"Requirement already satisfied: pandas in ./.venv/lib/python3.13/site-packages (2.2.3)\n",
|
|
"Collecting beautifulsoup4 (from gdown)\n",
|
|
" Using cached beautifulsoup4-4.12.3-py3-none-any.whl.metadata (3.8 kB)\n",
|
|
"Collecting filelock (from gdown)\n",
|
|
" Using cached filelock-3.16.1-py3-none-any.whl.metadata (2.9 kB)\n",
|
|
"Collecting requests[socks] (from gdown)\n",
|
|
" Using cached requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)\n",
|
|
"Collecting tqdm (from gdown)\n",
|
|
" Using cached tqdm-4.67.1-py3-none-any.whl.metadata (57 kB)\n",
|
|
"Requirement already satisfied: numpy>=1.26.0 in ./.venv/lib/python3.13/site-packages (from pandas) (2.2.1)\n",
|
|
"Requirement already satisfied: python-dateutil>=2.8.2 in ./.venv/lib/python3.13/site-packages (from pandas) (2.9.0.post0)\n",
|
|
"Requirement already satisfied: pytz>=2020.1 in ./.venv/lib/python3.13/site-packages (from pandas) (2024.2)\n",
|
|
"Requirement already satisfied: tzdata>=2022.7 in ./.venv/lib/python3.13/site-packages (from pandas) (2024.2)\n",
|
|
"Requirement already satisfied: six>=1.5 in ./.venv/lib/python3.13/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n",
|
|
"Collecting soupsieve>1.2 (from beautifulsoup4->gdown)\n",
|
|
" Using cached soupsieve-2.6-py3-none-any.whl.metadata (4.6 kB)\n",
|
|
"Collecting charset-normalizer<4,>=2 (from requests[socks]->gdown)\n",
|
|
" Downloading charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl.metadata (35 kB)\n",
|
|
"Collecting idna<4,>=2.5 (from requests[socks]->gdown)\n",
|
|
" Using cached idna-3.10-py3-none-any.whl.metadata (10 kB)\n",
|
|
"Collecting urllib3<3,>=1.21.1 (from requests[socks]->gdown)\n",
|
|
" Downloading urllib3-2.3.0-py3-none-any.whl.metadata (6.5 kB)\n",
|
|
"Collecting certifi>=2017.4.17 (from requests[socks]->gdown)\n",
|
|
" Downloading certifi-2024.12.14-py3-none-any.whl.metadata (2.3 kB)\n",
|
|
"Collecting PySocks!=1.5.7,>=1.5.6 (from requests[socks]->gdown)\n",
|
|
" Using cached PySocks-1.7.1-py3-none-any.whl.metadata (13 kB)\n",
|
|
"Using cached gdown-5.2.0-py3-none-any.whl (18 kB)\n",
|
|
"Using cached beautifulsoup4-4.12.3-py3-none-any.whl (147 kB)\n",
|
|
"Using cached filelock-3.16.1-py3-none-any.whl (16 kB)\n",
|
|
"Using cached tqdm-4.67.1-py3-none-any.whl (78 kB)\n",
|
|
"Downloading certifi-2024.12.14-py3-none-any.whl (164 kB)\n",
|
|
"Downloading charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl (195 kB)\n",
|
|
"Using cached idna-3.10-py3-none-any.whl (70 kB)\n",
|
|
"Using cached PySocks-1.7.1-py3-none-any.whl (16 kB)\n",
|
|
"Using cached soupsieve-2.6-py3-none-any.whl (36 kB)\n",
|
|
"Downloading urllib3-2.3.0-py3-none-any.whl (128 kB)\n",
|
|
"Using cached requests-2.32.3-py3-none-any.whl (64 kB)\n",
|
|
"Installing collected packages: urllib3, tqdm, soupsieve, PySocks, idna, filelock, charset-normalizer, certifi, requests, beautifulsoup4, gdown\n",
|
|
"Successfully installed PySocks-1.7.1 beautifulsoup4-4.12.3 certifi-2024.12.14 charset-normalizer-3.4.1 filelock-3.16.1 gdown-5.2.0 idna-3.10 requests-2.32.3 soupsieve-2.6 tqdm-4.67.1 urllib3-2.3.0\n",
|
|
"\n",
|
|
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
|
|
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
|
|
"Note: you may need to restart the kernel to use updated packages.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"######## Installations - BE SURE TO MAKE YOUR OWN LOCAL VENV FIRST\n",
|
|
"\n",
|
|
"%pip install gdown pandas"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {
|
|
"id": "KgzveXyAp35v"
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"######## Imports\n",
|
|
"\n",
|
|
"import pandas as pd\n",
|
|
"import gdown\n",
|
|
"import tarfile\n",
|
|
"import os\n",
|
|
"import json\n",
|
|
"from datetime import datetime\n",
|
|
"import csv"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {
|
|
"colab": {
|
|
"base_uri": "https://localhost:8080/"
|
|
},
|
|
"id": "iwcBxgxgqES8",
|
|
"outputId": "4af8de17-57bd-4857-f49c-2ee5d39ed248"
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"'./longmemeval_data/longmemeval_data.tar.gz' already exists, skipping download.\n",
|
|
"'longmemeval_oracle.json' already exists, so skipping extraction.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"######## Download the eval dataset from the official Google Drive source\n",
|
|
"\n",
|
|
"file_id = '1zJgtYRFhOh5zDQzzatiddfjYhFSnyQ80'\n",
|
|
"url = f'https://drive.google.com/uc?id={file_id}'\n",
|
|
"folder_path = './longmemeval_data'\n",
|
|
"file_path = os.path.join(folder_path, 'longmemeval_data.tar.gz')\n",
|
|
"\n",
|
|
"# If it doesn't exist, create a \"/.longmemeval_data/\" directory\n",
|
|
"if not os.path.exists(folder_path):\n",
|
|
" os.makedirs(folder_path)\n",
|
|
"\n",
|
|
"# Download the compressed dataset\n",
|
|
"if not os.path.exists(file_path):\n",
|
|
" gdown.download(url, file_path, quiet=False)\n",
|
|
"else:\n",
|
|
" print(f\"'{file_path}' already exists, skipping download.\")\n",
|
|
"\n",
|
|
"# Extract the tar.gz\n",
|
|
"if not os.path.exists(os.path.join(folder_path, 'longmemeval_oracle.json')):\n",
|
|
" with tarfile.open(file_path, 'r:gz') as tar:\n",
|
|
" tar.extractall(path=folder_path)\n",
|
|
"else:\n",
|
|
" print(\"'longmemeval_oracle.json' already exists, so skipping extraction.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {
|
|
"id": "_RjEZnk5v530"
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>question_id</th>\n",
|
|
" <th>question_type</th>\n",
|
|
" <th>question</th>\n",
|
|
" <th>answer</th>\n",
|
|
" <th>question_date</th>\n",
|
|
" <th>haystack_dates</th>\n",
|
|
" <th>haystack_session_ids</th>\n",
|
|
" <th>haystack_sessions</th>\n",
|
|
" <th>answer_session_ids</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>What was the first issue I had with my new car...</td>\n",
|
|
" <td>GPS system not functioning correctly</td>\n",
|
|
" <td>2023/04/10 (Mon) 23:07</td>\n",
|
|
" <td>[2023/04/10 (Mon) 17:50, 2023/04/10 (Mon) 14:4...</td>\n",
|
|
" <td>[answer_4be1b6b4_2, answer_4be1b6b4_3, answer_...</td>\n",
|
|
" <td>[[{'role': 'user', 'content': 'I'm thinking of...</td>\n",
|
|
" <td>[answer_4be1b6b4_2, answer_4be1b6b4_3, answer_...</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>gpt4_2487a7cb</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>Which event did I attend first, the 'Effective...</td>\n",
|
|
" <td>'Data Analysis using Python' webinar</td>\n",
|
|
" <td>2023/05/28 (Sun) 06:47</td>\n",
|
|
" <td>[2023/05/28 (Sun) 21:04, 2023/05/28 (Sun) 07:17]</td>\n",
|
|
" <td>[answer_1c6b85ea_1, answer_1c6b85ea_2]</td>\n",
|
|
" <td>[[{'role': 'user', 'content': 'I'm trying to g...</td>\n",
|
|
" <td>[answer_1c6b85ea_1, answer_1c6b85ea_2]</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>gpt4_76048e76</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>Which vehicle did I take care of first in Febr...</td>\n",
|
|
" <td>bike</td>\n",
|
|
" <td>2023/03/10 (Fri) 23:15</td>\n",
|
|
" <td>[2023/03/10 (Fri) 22:50, 2023/03/10 (Fri) 08:11]</td>\n",
|
|
" <td>[answer_b535969f_2, answer_b535969f_1]</td>\n",
|
|
" <td>[[{'role': 'user', 'content': 'I'm thinking of...</td>\n",
|
|
" <td>[answer_b535969f_2, answer_b535969f_1]</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>gpt4_2312f94c</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>Which device did I got first, the Samsung Gala...</td>\n",
|
|
" <td>Samsung Galaxy S22</td>\n",
|
|
" <td>2023/03/15 (Wed) 03:53</td>\n",
|
|
" <td>[2023/03/15 (Wed) 00:56, 2023/03/15 (Wed) 10:31]</td>\n",
|
|
" <td>[answer_5328c3c2_1, answer_5328c3c2_2]</td>\n",
|
|
" <td>[[{'role': 'user', 'content': 'I'm planning a ...</td>\n",
|
|
" <td>[answer_5328c3c2_1, answer_5328c3c2_2]</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>0bb5a684</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>How many days before the team meeting I was pr...</td>\n",
|
|
" <td>7 days. 8 days (including the last day) is als...</td>\n",
|
|
" <td>2023/01/13 (Fri) 19:39</td>\n",
|
|
" <td>[2023/01/13 (Fri) 18:07, 2023/01/13 (Fri) 21:38]</td>\n",
|
|
" <td>[answer_e936197f_1, answer_e936197f_2]</td>\n",
|
|
" <td>[[{'role': 'user', 'content': 'I'm preparing f...</td>\n",
|
|
" <td>[answer_e936197f_1, answer_e936197f_2]</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" question_id question_type \\\n",
|
|
"0 gpt4_2655b836 temporal-reasoning \n",
|
|
"1 gpt4_2487a7cb temporal-reasoning \n",
|
|
"2 gpt4_76048e76 temporal-reasoning \n",
|
|
"3 gpt4_2312f94c temporal-reasoning \n",
|
|
"4 0bb5a684 temporal-reasoning \n",
|
|
"\n",
|
|
" question \\\n",
|
|
"0 What was the first issue I had with my new car... \n",
|
|
"1 Which event did I attend first, the 'Effective... \n",
|
|
"2 Which vehicle did I take care of first in Febr... \n",
|
|
"3 Which device did I got first, the Samsung Gala... \n",
|
|
"4 How many days before the team meeting I was pr... \n",
|
|
"\n",
|
|
" answer question_date \\\n",
|
|
"0 GPS system not functioning correctly 2023/04/10 (Mon) 23:07 \n",
|
|
"1 'Data Analysis using Python' webinar 2023/05/28 (Sun) 06:47 \n",
|
|
"2 bike 2023/03/10 (Fri) 23:15 \n",
|
|
"3 Samsung Galaxy S22 2023/03/15 (Wed) 03:53 \n",
|
|
"4 7 days. 8 days (including the last day) is als... 2023/01/13 (Fri) 19:39 \n",
|
|
"\n",
|
|
" haystack_dates \\\n",
|
|
"0 [2023/04/10 (Mon) 17:50, 2023/04/10 (Mon) 14:4... \n",
|
|
"1 [2023/05/28 (Sun) 21:04, 2023/05/28 (Sun) 07:17] \n",
|
|
"2 [2023/03/10 (Fri) 22:50, 2023/03/10 (Fri) 08:11] \n",
|
|
"3 [2023/03/15 (Wed) 00:56, 2023/03/15 (Wed) 10:31] \n",
|
|
"4 [2023/01/13 (Fri) 18:07, 2023/01/13 (Fri) 21:38] \n",
|
|
"\n",
|
|
" haystack_session_ids \\\n",
|
|
"0 [answer_4be1b6b4_2, answer_4be1b6b4_3, answer_... \n",
|
|
"1 [answer_1c6b85ea_1, answer_1c6b85ea_2] \n",
|
|
"2 [answer_b535969f_2, answer_b535969f_1] \n",
|
|
"3 [answer_5328c3c2_1, answer_5328c3c2_2] \n",
|
|
"4 [answer_e936197f_1, answer_e936197f_2] \n",
|
|
"\n",
|
|
" haystack_sessions \\\n",
|
|
"0 [[{'role': 'user', 'content': 'I'm thinking of... \n",
|
|
"1 [[{'role': 'user', 'content': 'I'm trying to g... \n",
|
|
"2 [[{'role': 'user', 'content': 'I'm thinking of... \n",
|
|
"3 [[{'role': 'user', 'content': 'I'm planning a ... \n",
|
|
"4 [[{'role': 'user', 'content': 'I'm preparing f... \n",
|
|
"\n",
|
|
" answer_session_ids \n",
|
|
"0 [answer_4be1b6b4_2, answer_4be1b6b4_3, answer_... \n",
|
|
"1 [answer_1c6b85ea_1, answer_1c6b85ea_2] \n",
|
|
"2 [answer_b535969f_2, answer_b535969f_1] \n",
|
|
"3 [answer_5328c3c2_1, answer_5328c3c2_2] \n",
|
|
"4 [answer_e936197f_1, answer_e936197f_2] "
|
|
]
|
|
},
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"######## Load the eval dataset\n",
|
|
"\n",
|
|
"lme_dataset_option = os.path.join(folder_path, 'longmemeval_oracle.json') # Can be _oracle, _s, or _m\n",
|
|
"lme_dataset_df = pd.read_json(lme_dataset_option)\n",
|
|
"lme_dataset_df.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 13,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"######## Method to save all of the snippets (or only firsts/lasts) of the specified multi-sessions to a CSV file\n",
|
|
"\n",
|
|
"\n",
|
|
"def snippetize_lme_dataset(lme_filename, max_num_previous_messages=5):\n",
|
|
" \"\"\"\n",
|
|
" Creates a csv where each row is a \"snippet\" from longmemeval. A snippet is a message and set of previous messages.\n",
|
|
" \"\"\"\n",
|
|
"\n",
|
|
" lme_dataset_option = os.path.join(folder_path, lme_filename)\n",
|
|
" lme_dataset_df = pd.read_json(lme_dataset_option)\n",
|
|
"\n",
|
|
" all_snippets = []\n",
|
|
" for index, row in lme_dataset_df.iterrows():\n",
|
|
"\n",
|
|
" question_id = row['question_id']\n",
|
|
"\n",
|
|
" # Extract the haystack_sessions and dates\n",
|
|
" sessions = row['haystack_sessions']\n",
|
|
" session_dates = row['haystack_dates']\n",
|
|
"\n",
|
|
" # Combine into list of dictionaries\n",
|
|
" sessions_data = [\n",
|
|
" {\n",
|
|
" \"session\": session,\n",
|
|
" \"date\": datetime.strptime(date, \"%Y/%m/%d (%a) %H:%M\")\n",
|
|
" } \n",
|
|
" for session, date in zip(sessions, session_dates)\n",
|
|
" ]\n",
|
|
"\n",
|
|
" # Sort by date from earliest to latest\n",
|
|
" sessions_data.sort(key=lambda x: x[\"date\"])\n",
|
|
"\n",
|
|
"\n",
|
|
" all_snippets_this_session = []\n",
|
|
"\n",
|
|
" message_index_across_sessions = 0\n",
|
|
" for session_index, session_and_date in enumerate(sessions_data):\n",
|
|
" for message_index_within_session, message in enumerate(session_and_date[\"session\"]):\n",
|
|
" \n",
|
|
" num_previous_messages = min(max_num_previous_messages, message_index_across_sessions)\n",
|
|
" previous_snippets = all_snippets_this_session[message_index_across_sessions-num_previous_messages:]\n",
|
|
" previous_messages_only = [{\"role\":previous_snippet[\"message\"][\"role\"], \"content\":previous_snippet[\"message\"][\"content\"]} for previous_snippet in previous_snippets]\n",
|
|
"\n",
|
|
" snippet = {\n",
|
|
" \"question_id\": question_id,\n",
|
|
" \"question_type\": row[\"question_type\"],\n",
|
|
" \"multisession_index\": index,\n",
|
|
" \"session_index\": session_index,\n",
|
|
" \"message_index_within_session\": message_index_within_session,\n",
|
|
" \"message_index_across_sessions\": message_index_across_sessions,\n",
|
|
" \"session_date\": session_and_date[\"date\"],\n",
|
|
" \"message\": message,\n",
|
|
" \"previous_messages\": previous_messages_only,\n",
|
|
" \"num_previous_messages\": num_previous_messages,\n",
|
|
" }\n",
|
|
"\n",
|
|
" if lme_filename == \"longmemeval_oracle.json\":\n",
|
|
" snippet[\"message_has_answer\"] = message[\"has_answer\"]\n",
|
|
"\n",
|
|
" all_snippets_this_session.append(snippet)\n",
|
|
" message_index_across_sessions += 1\n",
|
|
"\n",
|
|
" all_snippets.extend(all_snippets_this_session)\n",
|
|
"\n",
|
|
" \n",
|
|
"\n",
|
|
" snippetized_folder = os.path.join(folder_path, \"snippetized_data\")\n",
|
|
" if not os.path.exists(snippetized_folder):\n",
|
|
" os.makedirs(snippetized_folder)\n",
|
|
"\n",
|
|
" filename = lme_filename.replace(\".json\", \"_snippetized.csv\")\n",
|
|
" filepath = os.path.join(snippetized_folder, filename)\n",
|
|
" \n",
|
|
" with open(filepath, \"w\", newline=\"\") as csvfile:\n",
|
|
" writer = csv.DictWriter(csvfile, fieldnames=all_snippets[0].keys())\n",
|
|
" writer.writeheader()\n",
|
|
" for snippet in all_snippets:\n",
|
|
" snippet['message'] = json.dumps(snippet['message'])\n",
|
|
" snippet['previous_messages'] = json.dumps(snippet['previous_messages'])\n",
|
|
" writer.writerow(snippet)\n",
|
|
"\n",
|
|
"\n",
|
|
"\n",
|
|
"\n",
|
|
"def snippetize_and_check(lme_filename):\n",
|
|
" folder_path = './longmemeval_data/snippetized_data'\n",
|
|
" file_path = os.path.join(folder_path, lme_filename.replace(\".json\", \"_snippetized.csv\"))\n",
|
|
" if not os.path.exists(file_path):\n",
|
|
" print(f\"Snippetizing {lme_filename}...\")\n",
|
|
" snippetize_lme_dataset(lme_filename)\n",
|
|
" else:\n",
|
|
" print(f\"Skipping snippetization for {lme_filename} because it already exists.\")\n",
|
|
"\n",
|
|
" # Check first few rows of the csv\n",
|
|
" df = pd.read_csv(file_path)\n",
|
|
" display(df.head(10))\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 14,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Snippetizing longmemeval_oracle.json...\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>question_id</th>\n",
|
|
" <th>question_type</th>\n",
|
|
" <th>multisession_index</th>\n",
|
|
" <th>session_index</th>\n",
|
|
" <th>message_index_within_session</th>\n",
|
|
" <th>message_index_across_sessions</th>\n",
|
|
" <th>session_date</th>\n",
|
|
" <th>message</th>\n",
|
|
" <th>previous_messages</th>\n",
|
|
" <th>num_previous_messages</th>\n",
|
|
" <th>message_has_answer</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>2023-04-10 14:47:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"I'm thinking of g...</td>\n",
|
|
" <td>[]</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>2023-04-10 14:47:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Choosing the...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"I'm thinking of ...</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>2023-04-10 14:47:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"I've been doing s...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"I'm thinking of ...</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>True</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>2023-04-10 14:47:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"That's great...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"I'm thinking of ...</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>4</td>\n",
|
|
" <td>4</td>\n",
|
|
" <td>2023-04-10 14:47:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"I'll definitely a...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"I'm thinking of ...</td>\n",
|
|
" <td>4</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>2023-04-10 14:47:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Advanced pai...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"I'm thinking of ...</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>6</td>\n",
|
|
" <td>6</td>\n",
|
|
" <td>2023-04-10 14:47:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"I'll definitely a...</td>\n",
|
|
" <td>[{\"role\": \"assistant\", \"content\": \"Choosing th...</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>7</td>\n",
|
|
" <td>7</td>\n",
|
|
" <td>2023-04-10 14:47:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Congratulati...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"I've been doing ...</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>8</td>\n",
|
|
" <td>8</td>\n",
|
|
" <td>2023-04-10 14:47:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"That's really hel...</td>\n",
|
|
" <td>[{\"role\": \"assistant\", \"content\": \"That's grea...</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>gpt4_2655b836</td>\n",
|
|
" <td>temporal-reasoning</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>9</td>\n",
|
|
" <td>9</td>\n",
|
|
" <td>2023-04-10 14:47:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Congratulati...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"I'll definitely ...</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>False</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" question_id question_type multisession_index session_index \\\n",
|
|
"0 gpt4_2655b836 temporal-reasoning 0 0 \n",
|
|
"1 gpt4_2655b836 temporal-reasoning 0 0 \n",
|
|
"2 gpt4_2655b836 temporal-reasoning 0 0 \n",
|
|
"3 gpt4_2655b836 temporal-reasoning 0 0 \n",
|
|
"4 gpt4_2655b836 temporal-reasoning 0 0 \n",
|
|
"5 gpt4_2655b836 temporal-reasoning 0 0 \n",
|
|
"6 gpt4_2655b836 temporal-reasoning 0 0 \n",
|
|
"7 gpt4_2655b836 temporal-reasoning 0 0 \n",
|
|
"8 gpt4_2655b836 temporal-reasoning 0 0 \n",
|
|
"9 gpt4_2655b836 temporal-reasoning 0 0 \n",
|
|
"\n",
|
|
" message_index_within_session message_index_across_sessions \\\n",
|
|
"0 0 0 \n",
|
|
"1 1 1 \n",
|
|
"2 2 2 \n",
|
|
"3 3 3 \n",
|
|
"4 4 4 \n",
|
|
"5 5 5 \n",
|
|
"6 6 6 \n",
|
|
"7 7 7 \n",
|
|
"8 8 8 \n",
|
|
"9 9 9 \n",
|
|
"\n",
|
|
" session_date message \\\n",
|
|
"0 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'm thinking of g... \n",
|
|
"1 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Choosing the... \n",
|
|
"2 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I've been doing s... \n",
|
|
"3 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"That's great... \n",
|
|
"4 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'll definitely a... \n",
|
|
"5 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Advanced pai... \n",
|
|
"6 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"I'll definitely a... \n",
|
|
"7 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Congratulati... \n",
|
|
"8 2023-04-10 14:47:00 {\"role\": \"user\", \"content\": \"That's really hel... \n",
|
|
"9 2023-04-10 14:47:00 {\"role\": \"assistant\", \"content\": \"Congratulati... \n",
|
|
"\n",
|
|
" previous_messages num_previous_messages \\\n",
|
|
"0 [] 0 \n",
|
|
"1 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 1 \n",
|
|
"2 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 2 \n",
|
|
"3 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 3 \n",
|
|
"4 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 4 \n",
|
|
"5 [{\"role\": \"user\", \"content\": \"I'm thinking of ... 5 \n",
|
|
"6 [{\"role\": \"assistant\", \"content\": \"Choosing th... 5 \n",
|
|
"7 [{\"role\": \"user\", \"content\": \"I've been doing ... 5 \n",
|
|
"8 [{\"role\": \"assistant\", \"content\": \"That's grea... 5 \n",
|
|
"9 [{\"role\": \"user\", \"content\": \"I'll definitely ... 5 \n",
|
|
"\n",
|
|
" message_has_answer \n",
|
|
"0 False \n",
|
|
"1 False \n",
|
|
"2 True \n",
|
|
"3 False \n",
|
|
"4 False \n",
|
|
"5 False \n",
|
|
"6 False \n",
|
|
"7 False \n",
|
|
"8 False \n",
|
|
"9 False "
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"lme_filename = \"longmemeval_oracle.json\"\n",
|
|
"snippetize_and_check(lme_filename)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 15,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Snippetizing longmemeval_s.json...\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>question_id</th>\n",
|
|
" <th>question_type</th>\n",
|
|
" <th>multisession_index</th>\n",
|
|
" <th>session_index</th>\n",
|
|
" <th>message_index_within_session</th>\n",
|
|
" <th>message_index_across_sessions</th>\n",
|
|
" <th>session_date</th>\n",
|
|
" <th>message</th>\n",
|
|
" <th>previous_messages</th>\n",
|
|
" <th>num_previous_messages</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>e47becba</td>\n",
|
|
" <td>single-session-user</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>2023-05-20 02:21:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"The farmer needs ...</td>\n",
|
|
" <td>[]</td>\n",
|
|
" <td>0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>e47becba</td>\n",
|
|
" <td>single-session-user</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>2023-05-20 02:21:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"To solve thi...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"The farmer needs...</td>\n",
|
|
" <td>1</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>e47becba</td>\n",
|
|
" <td>single-session-user</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>2023-05-20 02:57:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"I'm trying to sta...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"The farmer needs...</td>\n",
|
|
" <td>2</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>e47becba</td>\n",
|
|
" <td>single-session-user</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>2023-05-20 02:57:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Congratulati...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"The farmer needs...</td>\n",
|
|
" <td>3</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>e47becba</td>\n",
|
|
" <td>single-session-user</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>4</td>\n",
|
|
" <td>2023-05-20 02:57:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"I've been doing s...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"The farmer needs...</td>\n",
|
|
" <td>4</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>e47becba</td>\n",
|
|
" <td>single-session-user</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>2023-05-20 02:57:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Yoga is an e...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"The farmer needs...</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>e47becba</td>\n",
|
|
" <td>single-session-user</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>4</td>\n",
|
|
" <td>6</td>\n",
|
|
" <td>2023-05-20 02:57:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"That's really hel...</td>\n",
|
|
" <td>[{\"role\": \"assistant\", \"content\": \"To solve th...</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>e47becba</td>\n",
|
|
" <td>single-session-user</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>7</td>\n",
|
|
" <td>2023-05-20 02:57:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Foam rolling...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"I'm trying to st...</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>e47becba</td>\n",
|
|
" <td>single-session-user</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>6</td>\n",
|
|
" <td>8</td>\n",
|
|
" <td>2023-05-20 02:57:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"I've also been tr...</td>\n",
|
|
" <td>[{\"role\": \"assistant\", \"content\": \"Congratulat...</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>e47becba</td>\n",
|
|
" <td>single-session-user</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>7</td>\n",
|
|
" <td>9</td>\n",
|
|
" <td>2023-05-20 02:57:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Monitoring y...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"I've been doing ...</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" question_id question_type multisession_index session_index \\\n",
|
|
"0 e47becba single-session-user 0 0 \n",
|
|
"1 e47becba single-session-user 0 0 \n",
|
|
"2 e47becba single-session-user 0 1 \n",
|
|
"3 e47becba single-session-user 0 1 \n",
|
|
"4 e47becba single-session-user 0 1 \n",
|
|
"5 e47becba single-session-user 0 1 \n",
|
|
"6 e47becba single-session-user 0 1 \n",
|
|
"7 e47becba single-session-user 0 1 \n",
|
|
"8 e47becba single-session-user 0 1 \n",
|
|
"9 e47becba single-session-user 0 1 \n",
|
|
"\n",
|
|
" message_index_within_session message_index_across_sessions \\\n",
|
|
"0 0 0 \n",
|
|
"1 1 1 \n",
|
|
"2 0 2 \n",
|
|
"3 1 3 \n",
|
|
"4 2 4 \n",
|
|
"5 3 5 \n",
|
|
"6 4 6 \n",
|
|
"7 5 7 \n",
|
|
"8 6 8 \n",
|
|
"9 7 9 \n",
|
|
"\n",
|
|
" session_date message \\\n",
|
|
"0 2023-05-20 02:21:00 {\"role\": \"user\", \"content\": \"The farmer needs ... \n",
|
|
"1 2023-05-20 02:21:00 {\"role\": \"assistant\", \"content\": \"To solve thi... \n",
|
|
"2 2023-05-20 02:57:00 {\"role\": \"user\", \"content\": \"I'm trying to sta... \n",
|
|
"3 2023-05-20 02:57:00 {\"role\": \"assistant\", \"content\": \"Congratulati... \n",
|
|
"4 2023-05-20 02:57:00 {\"role\": \"user\", \"content\": \"I've been doing s... \n",
|
|
"5 2023-05-20 02:57:00 {\"role\": \"assistant\", \"content\": \"Yoga is an e... \n",
|
|
"6 2023-05-20 02:57:00 {\"role\": \"user\", \"content\": \"That's really hel... \n",
|
|
"7 2023-05-20 02:57:00 {\"role\": \"assistant\", \"content\": \"Foam rolling... \n",
|
|
"8 2023-05-20 02:57:00 {\"role\": \"user\", \"content\": \"I've also been tr... \n",
|
|
"9 2023-05-20 02:57:00 {\"role\": \"assistant\", \"content\": \"Monitoring y... \n",
|
|
"\n",
|
|
" previous_messages num_previous_messages \n",
|
|
"0 [] 0 \n",
|
|
"1 [{\"role\": \"user\", \"content\": \"The farmer needs... 1 \n",
|
|
"2 [{\"role\": \"user\", \"content\": \"The farmer needs... 2 \n",
|
|
"3 [{\"role\": \"user\", \"content\": \"The farmer needs... 3 \n",
|
|
"4 [{\"role\": \"user\", \"content\": \"The farmer needs... 4 \n",
|
|
"5 [{\"role\": \"user\", \"content\": \"The farmer needs... 5 \n",
|
|
"6 [{\"role\": \"assistant\", \"content\": \"To solve th... 5 \n",
|
|
"7 [{\"role\": \"user\", \"content\": \"I'm trying to st... 5 \n",
|
|
"8 [{\"role\": \"assistant\", \"content\": \"Congratulat... 5 \n",
|
|
"9 [{\"role\": \"user\", \"content\": \"I've been doing ... 5 "
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"lme_filename = \"longmemeval_s.json\"\n",
|
|
"snippetize_and_check(lme_filename)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 16,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Snippetizing longmemeval_m.json...\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div>\n",
|
|
"<style scoped>\n",
|
|
" .dataframe tbody tr th:only-of-type {\n",
|
|
" vertical-align: middle;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe tbody tr th {\n",
|
|
" vertical-align: top;\n",
|
|
" }\n",
|
|
"\n",
|
|
" .dataframe thead th {\n",
|
|
" text-align: right;\n",
|
|
" }\n",
|
|
"</style>\n",
|
|
"<table border=\"1\" class=\"dataframe\">\n",
|
|
" <thead>\n",
|
|
" <tr style=\"text-align: right;\">\n",
|
|
" <th></th>\n",
|
|
" <th>question_id</th>\n",
|
|
" <th>question_type</th>\n",
|
|
" <th>multisession_index</th>\n",
|
|
" <th>session_index</th>\n",
|
|
" <th>message_index_within_session</th>\n",
|
|
" <th>message_index_across_sessions</th>\n",
|
|
" <th>session_date</th>\n",
|
|
" <th>message</th>\n",
|
|
" <th>previous_messages</th>\n",
|
|
" <th>num_previous_messages</th>\n",
|
|
" </tr>\n",
|
|
" </thead>\n",
|
|
" <tbody>\n",
|
|
" <tr>\n",
|
|
" <th>0</th>\n",
|
|
" <td>7161e7e2</td>\n",
|
|
" <td>single-session-assistant</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>2023-05-20 00:04:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"Can you provide t...</td>\n",
|
|
" <td>[]</td>\n",
|
|
" <td>0</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>1</th>\n",
|
|
" <td>7161e7e2</td>\n",
|
|
" <td>single-session-assistant</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>2023-05-20 00:04:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Yes, here ar...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"Can you provide ...</td>\n",
|
|
" <td>1</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>2</th>\n",
|
|
" <td>7161e7e2</td>\n",
|
|
" <td>single-session-assistant</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>2023-05-20 00:04:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"Wow, it's amazing...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"Can you provide ...</td>\n",
|
|
" <td>2</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>3</th>\n",
|
|
" <td>7161e7e2</td>\n",
|
|
" <td>single-session-assistant</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>2023-05-20 00:04:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Yes, achievi...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"Can you provide ...</td>\n",
|
|
" <td>3</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>4</th>\n",
|
|
" <td>7161e7e2</td>\n",
|
|
" <td>single-session-assistant</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>4</td>\n",
|
|
" <td>4</td>\n",
|
|
" <td>2023-05-20 00:04:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"It's crazy to thi...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"Can you provide ...</td>\n",
|
|
" <td>4</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>5</th>\n",
|
|
" <td>7161e7e2</td>\n",
|
|
" <td>single-session-assistant</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>5</td>\n",
|
|
" <td>2023-05-20 00:04:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Astronauts u...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"Can you provide ...</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>6</th>\n",
|
|
" <td>7161e7e2</td>\n",
|
|
" <td>single-session-assistant</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>6</td>\n",
|
|
" <td>2023-05-20 00:16:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"What about aliena...</td>\n",
|
|
" <td>[{\"role\": \"assistant\", \"content\": \"Yes, here a...</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>7</th>\n",
|
|
" <td>7161e7e2</td>\n",
|
|
" <td>single-session-assistant</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>7</td>\n",
|
|
" <td>2023-05-20 00:16:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"Alienation i...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"Wow, it's amazin...</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>8</th>\n",
|
|
" <td>7161e7e2</td>\n",
|
|
" <td>single-session-assistant</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>2</td>\n",
|
|
" <td>8</td>\n",
|
|
" <td>2023-05-20 00:16:00</td>\n",
|
|
" <td>{\"role\": \"user\", \"content\": \"Why you did not m...</td>\n",
|
|
" <td>[{\"role\": \"assistant\", \"content\": \"Yes, achiev...</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" <tr>\n",
|
|
" <th>9</th>\n",
|
|
" <td>7161e7e2</td>\n",
|
|
" <td>single-session-assistant</td>\n",
|
|
" <td>0</td>\n",
|
|
" <td>1</td>\n",
|
|
" <td>3</td>\n",
|
|
" <td>9</td>\n",
|
|
" <td>2023-05-20 00:16:00</td>\n",
|
|
" <td>{\"role\": \"assistant\", \"content\": \"I apologize ...</td>\n",
|
|
" <td>[{\"role\": \"user\", \"content\": \"It's crazy to th...</td>\n",
|
|
" <td>5</td>\n",
|
|
" </tr>\n",
|
|
" </tbody>\n",
|
|
"</table>\n",
|
|
"</div>"
|
|
],
|
|
"text/plain": [
|
|
" question_id question_type multisession_index session_index \\\n",
|
|
"0 7161e7e2 single-session-assistant 0 0 \n",
|
|
"1 7161e7e2 single-session-assistant 0 0 \n",
|
|
"2 7161e7e2 single-session-assistant 0 0 \n",
|
|
"3 7161e7e2 single-session-assistant 0 0 \n",
|
|
"4 7161e7e2 single-session-assistant 0 0 \n",
|
|
"5 7161e7e2 single-session-assistant 0 0 \n",
|
|
"6 7161e7e2 single-session-assistant 0 1 \n",
|
|
"7 7161e7e2 single-session-assistant 0 1 \n",
|
|
"8 7161e7e2 single-session-assistant 0 1 \n",
|
|
"9 7161e7e2 single-session-assistant 0 1 \n",
|
|
"\n",
|
|
" message_index_within_session message_index_across_sessions \\\n",
|
|
"0 0 0 \n",
|
|
"1 1 1 \n",
|
|
"2 2 2 \n",
|
|
"3 3 3 \n",
|
|
"4 4 4 \n",
|
|
"5 5 5 \n",
|
|
"6 0 6 \n",
|
|
"7 1 7 \n",
|
|
"8 2 8 \n",
|
|
"9 3 9 \n",
|
|
"\n",
|
|
" session_date message \\\n",
|
|
"0 2023-05-20 00:04:00 {\"role\": \"user\", \"content\": \"Can you provide t... \n",
|
|
"1 2023-05-20 00:04:00 {\"role\": \"assistant\", \"content\": \"Yes, here ar... \n",
|
|
"2 2023-05-20 00:04:00 {\"role\": \"user\", \"content\": \"Wow, it's amazing... \n",
|
|
"3 2023-05-20 00:04:00 {\"role\": \"assistant\", \"content\": \"Yes, achievi... \n",
|
|
"4 2023-05-20 00:04:00 {\"role\": \"user\", \"content\": \"It's crazy to thi... \n",
|
|
"5 2023-05-20 00:04:00 {\"role\": \"assistant\", \"content\": \"Astronauts u... \n",
|
|
"6 2023-05-20 00:16:00 {\"role\": \"user\", \"content\": \"What about aliena... \n",
|
|
"7 2023-05-20 00:16:00 {\"role\": \"assistant\", \"content\": \"Alienation i... \n",
|
|
"8 2023-05-20 00:16:00 {\"role\": \"user\", \"content\": \"Why you did not m... \n",
|
|
"9 2023-05-20 00:16:00 {\"role\": \"assistant\", \"content\": \"I apologize ... \n",
|
|
"\n",
|
|
" previous_messages num_previous_messages \n",
|
|
"0 [] 0 \n",
|
|
"1 [{\"role\": \"user\", \"content\": \"Can you provide ... 1 \n",
|
|
"2 [{\"role\": \"user\", \"content\": \"Can you provide ... 2 \n",
|
|
"3 [{\"role\": \"user\", \"content\": \"Can you provide ... 3 \n",
|
|
"4 [{\"role\": \"user\", \"content\": \"Can you provide ... 4 \n",
|
|
"5 [{\"role\": \"user\", \"content\": \"Can you provide ... 5 \n",
|
|
"6 [{\"role\": \"assistant\", \"content\": \"Yes, here a... 5 \n",
|
|
"7 [{\"role\": \"user\", \"content\": \"Wow, it's amazin... 5 \n",
|
|
"8 [{\"role\": \"assistant\", \"content\": \"Yes, achiev... 5 \n",
|
|
"9 [{\"role\": \"user\", \"content\": \"It's crazy to th... 5 "
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"lme_filename = \"longmemeval_m.json\"\n",
|
|
"snippetize_and_check(lme_filename)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"colab": {
|
|
"provenance": []
|
|
},
|
|
"kernelspec": {
|
|
"display_name": "Python (.venv)",
|
|
"language": "python",
|
|
"name": ".venv"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.13.0"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 0
|
|
}
|