From 3e654dfd5194b69602727d85729da449b7d0f2fa Mon Sep 17 00:00:00 2001 From: burnash Date: Thu, 17 Aug 2023 10:43:16 +0200 Subject: [PATCH 01/13] Add Python.gitignore --- .gitignore | 160 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..6769e21d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file From a61640bc488ecfcd4820aa51a40abcd809189946 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Thu, 17 Aug 2023 18:26:30 +0200 Subject: [PATCH 02/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 308e7801d..649624a6e 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ The rise of this new profession is perhaps signaling the need for a solution tha To address this issue, [dlthub](https://dlthub.com/) and [prometh.ai](http://prometh.ai/) will collaborate on a productionizing a common use-case, progressing step by step. We will utilize the LLMs, frameworks, and services, refining the code until we attain a clearer understanding of what a modern LLM architecture stack might entail. -## Read more on our blog post [prometh.ai](http://prometh.ai/promethai-memory-blog-post-on) +## Read more on our blog post [prometh.ai](http://prometh.ai/promethai-memory-blog-post-one) # PromethAI-Memory Repo Structure From 29a9baa0d1c7f0c96d678282e67c877eff8fefcb Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Thu, 17 Aug 2023 18:38:08 +0200 Subject: [PATCH 03/13] Delete .env --- level_2/.env | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 level_2/.env diff --git a/level_2/.env b/level_2/.env deleted file mode 100644 index 8e221f167..000000000 --- a/level_2/.env +++ /dev/null @@ -1,11 +0,0 @@ - OPENAI_API_KEY = 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 - PINECONE_API_KEY = 4e0982ad-57d5-41ac-bce5-d1fd2c2da273 - PINECONE_API_ENV = us-west1-gcp - REPLICATE_API_TOKEN = 4e0982ad-57d5-41ac-bce5-d1fd2c2da273 - GPLACES_API_KEY = AIzaSyAfuT9tBy6wC3phZR1Tl5acknNA_TU2mKE - REDIS_HOST=redis - SERPAPI_API_KEY=17bb94b76b0d7cf3fb1c36d8376e0fc4c3ed761e862b05ef154e116d73c39da5 - ZAPIER_NLA_API_KEY=sk-ak-GtXls7Y5JcPOSbWw7SZDzSvtAF - LOCAL_DEV = True - WEAVIATE_API_KEY =shCCL5EVpOKxIdZhMRH090lFqDb5aE1XgUTP - WEAVIATE_URL = https://new-test-cluster-i49dzudl.weaviate.network \ No newline at end of file From b5a86ca49472c8a8edc28fcdf83993df301a982e Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Fri, 25 Aug 2023 12:41:31 +0200 Subject: [PATCH 04/13] Update README.md --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 649624a6e..6c27566da 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Memory management for the AI Applications and AI Agents ![Infographic Image](https://github.com/topoteretes/PromethAI-Memory/blob/main/infographic_final.png) -## A journey toward a production-ready modern data platform +## The Motivation Browsing the database of theresanaiforthat.com, we can observe around [7000 new, mostly semi-finished projects](https://theresanaiforthat.com/) in the field of applied AI, whose development is fueled by new improvements in foundation models and open-source community contributions. @@ -26,10 +26,11 @@ The rise of this new profession is perhaps signaling the need for a solution tha To address this issue, [dlthub](https://dlthub.com/) and [prometh.ai](http://prometh.ai/) will collaborate on a productionizing a common use-case, progressing step by step. We will utilize the LLMs, frameworks, and services, refining the code until we attain a clearer understanding of what a modern LLM architecture stack might entail. -## Read more on our blog post [prometh.ai](http://prometh.ai/promethai-memory-blog-post-one) +### Read more on our blog post [prometh.ai](http://prometh.ai/promethai-memory-blog-post-one) # PromethAI-Memory Repo Structure -The repository contains a set of folders that represent the steps in the evolution of the modern data stack from POC towards production -Level 1 - CMD script to process PDFs +The repository contains a set of folders that represent the steps in the evolution of the modern data stack from POC to production +- Level 1 - CMD script to process PDFs +- Level 2 - Memory Manager implemented in Python From 79f6b151d1b05a8e6e740e830899351c61b0ed24 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Fri, 25 Aug 2023 12:42:21 +0200 Subject: [PATCH 05/13] Update README.md --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index 6c27566da..ef1a3f04d 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,6 @@ Decades of technological advancements have led to small teams being able to do i Yet, the AI apps currently being pushed out still mostly feel and perform like demos. -The consensus is, nevertheless, that the AI space is *the* place to be in 2023. - -> “The AI Engineer [...] will likely be the **highest-demand engineering job of the [coming] decade.”** -> - -**[Swyx](https://www.latent.space/p/ai-engineer)** - The rise of this new profession is perhaps signaling the need for a solution that is not yet there — a solution that in its essence represents a Large Language Model (LLM) — [a powerful general problem solver](https://lilianweng.github.io/posts/2023-06-23-agent/?fbclid=IwAR1p0W-Mg_4WtjOCeE8E6s7pJZlTDCDLmcXqHYVIrEVisz_D_S8LfN6Vv20) — available in the palm of your hand 24/7/365. To address this issue, [dlthub](https://dlthub.com/) and [prometh.ai](http://prometh.ai/) will collaborate on a productionizing a common use-case, progressing step by step. We will utilize the LLMs, frameworks, and services, refining the code until we attain a clearer understanding of what a modern LLM architecture stack might entail. From 23fc3b1b28cb708d144fa20f91b4b459cbb4dba9 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Sat, 26 Aug 2023 17:28:10 +0200 Subject: [PATCH 06/13] Update README.md --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index ef1a3f04d..5d9b6234a 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,7 @@ Browsing the database of theresanaiforthat.com, we can observe around [7000 new, It seems it has never been easier to create a startup, build an app, and go to market… and fail. -Decades of technological advancements have led to small teams being able to do in 2023 what in 2015 required a team of dozens. - -Yet, the AI apps currently being pushed out still mostly feel and perform like demos. - -The rise of this new profession is perhaps signaling the need for a solution that is not yet there — a solution that in its essence represents a Large Language Model (LLM) — [a powerful general problem solver](https://lilianweng.github.io/posts/2023-06-23-agent/?fbclid=IwAR1p0W-Mg_4WtjOCeE8E6s7pJZlTDCDLmcXqHYVIrEVisz_D_S8LfN6Vv20) — available in the palm of your hand 24/7/365. +AI apps currently being pushed out still mostly feel and perform like demos. To address this issue, [dlthub](https://dlthub.com/) and [prometh.ai](http://prometh.ai/) will collaborate on a productionizing a common use-case, progressing step by step. We will utilize the LLMs, frameworks, and services, refining the code until we attain a clearer understanding of what a modern LLM architecture stack might entail. From 05ca523710bf1bae0134c1548d3dc986cb4bb4a4 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Sat, 26 Aug 2023 17:33:31 +0200 Subject: [PATCH 07/13] Update README.md --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5d9b6234a..4773b88d9 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,27 @@ To address this issue, [dlthub](https://dlthub.com/) and [prometh.ai](http://pro ### Read more on our blog post [prometh.ai](http://prometh.ai/promethai-memory-blog-post-one) -# PromethAI-Memory Repo Structure +## PromethAI-Memory Repo Structure The repository contains a set of folders that represent the steps in the evolution of the modern data stack from POC to production - Level 1 - CMD script to process PDFs + We introduce the following concepts: + 1. Structured output with Pydantic + 2. CMD script to process custom PDFs - Level 2 - Memory Manager implemented in Python + We introduce the following concepts: + 1. Long Term Memory + 2. Short Term Memory + 3. Episodic Buffer + 4. Attention Modulators + The code at this level contains: + 1. Simple PDF ingestion + 2. FastAPI + 3. Docker Image + 4. Memory manager + 5. Langchain-based Agent Simulator + 6. Data schema + +## How to use + +Each of the folders contains a README to get started. From 8c2053db5bf5d3c561ca4f619e0b0c1bcddedbf3 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Sat, 26 Aug 2023 17:34:09 +0200 Subject: [PATCH 08/13] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4773b88d9..0a4a85edf 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,14 @@ The repository contains a set of folders that represent the steps in the evoluti 1. Structured output with Pydantic 2. CMD script to process custom PDFs - Level 2 - Memory Manager implemented in Python - We introduce the following concepts: + +We introduce the following concepts: 1. Long Term Memory 2. Short Term Memory 3. Episodic Buffer 4. Attention Modulators - The code at this level contains: + +The code at this level contains: 1. Simple PDF ingestion 2. FastAPI 3. Docker Image From 719e30e0b22bbb5e6745fe6586d139b435da6d65 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Sat, 26 Aug 2023 17:35:00 +0200 Subject: [PATCH 09/13] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0a4a85edf..81cb21a40 100644 --- a/README.md +++ b/README.md @@ -21,11 +21,13 @@ To address this issue, [dlthub](https://dlthub.com/) and [prometh.ai](http://pro ## PromethAI-Memory Repo Structure The repository contains a set of folders that represent the steps in the evolution of the modern data stack from POC to production -- Level 1 - CMD script to process PDFs + +#### Level 1 - CMD script to process PDFs We introduce the following concepts: 1. Structured output with Pydantic 2. CMD script to process custom PDFs -- Level 2 - Memory Manager implemented in Python + +#### Level 2 - Memory Manager implemented in Python We introduce the following concepts: 1. Long Term Memory From 3cdc75ed65e7368692540f46e0f0d65968fd6003 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Mon, 4 Sep 2023 20:35:01 +0200 Subject: [PATCH 10/13] Update README.md --- README.md | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/README.md b/README.md index 81cb21a40..ea488410d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,73 @@ Memory management for the AI Applications and AI Agents +

+ + promethAI logo + + + +

+ +

Open-source framework that manages memory for AI Agents and LLM apps

+ +

+ +promethAI forks + + + +promethAI stars + + +promethAI pull-requests + + + + + +

+ +[//]: # (

Follow PromethAI

) + +[//]: # (

) + +[//]: # () + +[//]: # (Follow _promethAI) + +[//]: # () + +[//]: # (

) + +[//]: # () + +[//]: # (

) + +[//]: # () + +[//]: # (

) + + +

Share promethAI Repository

+ +

+ + +Follow _promethAI +Share on Telegram + +Share on Reddit + Buy Me A Coffee + +

+ +
+ + + + ![Infographic Image](https://github.com/topoteretes/PromethAI-Memory/blob/main/infographic_final.png) ## The Motivation From d1b314de396a3e4fc08cd9e0e6205d6126d18fd3 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Mon, 4 Sep 2023 20:35:27 +0200 Subject: [PATCH 11/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ea488410d..04ad848ed 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # PromethAI-Memory -Memory management for the AI Applications and AI Agents + From cf243ad872e42a0baf8fa2eb98df3f5d4d480398 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 6 Sep 2023 17:18:14 +0200 Subject: [PATCH 12/13] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 04ad848ed..dd741203e 100644 --- a/README.md +++ b/README.md @@ -80,10 +80,14 @@ It seems it has never been easier to create a startup, build an app, and go to m AI apps currently being pushed out still mostly feel and perform like demos. -To address this issue, [dlthub](https://dlthub.com/) and [prometh.ai](http://prometh.ai/) will collaborate on a productionizing a common use-case, progressing step by step. We will utilize the LLMs, frameworks, and services, refining the code until we attain a clearer understanding of what a modern LLM architecture stack might entail. +To address this issue, [dlthub](https://dlthub.com/) and [prometh.ai](http://prometh.ai/) will collaborate on productionizing a common use-case, progressing step by step. We will utilize the LLMs, frameworks, and services, refining the code until we attain a clearer understanding of what a modern LLM architecture stack might entail. + +We go on a journey and propose a new way to reason about Language Architecture for Agents based on cognitive sciences. ### Read more on our blog post [prometh.ai](http://prometh.ai/promethai-memory-blog-post-one) +### Or check this [Princeton paper](https://arxiv.org/abs/2309.02427) released after our demo + ## PromethAI-Memory Repo Structure From 23a0437fbdbcceeb1af40e3bff81e8d3707214b9 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 6 Sep 2023 17:18:59 +0200 Subject: [PATCH 13/13] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dd741203e..079eddac9 100644 --- a/README.md +++ b/README.md @@ -84,9 +84,9 @@ To address this issue, [dlthub](https://dlthub.com/) and [prometh.ai](http://pro We go on a journey and propose a new way to reason about Language Architecture for Agents based on cognitive sciences. -### Read more on our blog post [prometh.ai](http://prometh.ai/promethai-memory-blog-post-one) +#### Read more on our blog post [prometh.ai](http://prometh.ai/promethai-memory-blog-post-one) -### Or check this [Princeton paper](https://arxiv.org/abs/2309.02427) released after our demo +#### Or check this [Princeton paper](https://arxiv.org/abs/2309.02427) released after our demo ## PromethAI-Memory Repo Structure