From c183742ad54a9933b40a8975f0a9d323427b76c5 Mon Sep 17 00:00:00 2001 From: Igor Ilic Date: Tue, 29 Oct 2024 13:47:23 +0100 Subject: [PATCH 1/8] test: Add test for Jupyter notebook Added testing of Jupyter notebook through github actions Test #COG-462 --- .github/workflows/test_notebook.yml | 58 +++++++++++++++++++++++++++++ notebooks/cognee_demo.ipynb | 10 +++-- 2 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/test_notebook.yml diff --git a/.github/workflows/test_notebook.yml b/.github/workflows/test_notebook.yml new file mode 100644 index 000000000..659714d94 --- /dev/null +++ b/.github/workflows/test_notebook.yml @@ -0,0 +1,58 @@ +name: test | notebook + +on: + pull_request: + branches: + - main + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + RUNTIME__LOG_LEVEL: ERROR + +jobs: + get_docs_changes: + name: docs changes + uses: ./.github/workflows/get_docs_changes.yml + + run_notebook_test: + name: test + needs: get_docs_changes + if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - name: Check out + uses: actions/checkout@master + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11.x' + + - name: Install Poetry + uses: snok/install-poetry@v1.3.2 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Install dependencies + run: poetry install --no-interaction + + - name: Execute Jupyter Notebook + env: + ENV: 'dev' + LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GRAPHISTRY_USERNAME: {{ secrets.GRAPHISTRY_USERNAME }} + GRAPHISTRY_PASSWORD: {{ secrets.GRAPHISTRY_PASSWORD }} + run: jupyter nbconvert \ + --to notebook \ + --execute notebooks/cognee_demo.ipynb \ + --output notebooks/executed_notebook.ipynb \ + --ExecutePreprocessor.timeout=1200 \ No newline at end of file diff --git a/notebooks/cognee_demo.ipynb b/notebooks/cognee_demo.ipynb index 8b8124490..c2c249538 100644 --- a/notebooks/cognee_demo.ipynb +++ b/notebooks/cognee_demo.ipynb @@ -537,10 +537,14 @@ "import os\n", "\n", "# # Setting environment variables\n", - "os.environ[\"GRAPHISTRY_USERNAME\"] = \"\"\n", - "os.environ[\"GRAPHISTRY_PASSWORD\"] = \"\"\n", + "if \"GRAPHISTRY_USERNAME\" not in os.environ: \n", + " os.environ[\"GRAPHISTRY_USERNAME\"] = \"\"\n", "\n", - "os.environ[\"LLM_API_KEY\"] = \"\"\n", + "if \"GRAPHISTRY_PASSWORD\" not in os.environ: \n", + " os.environ[\"GRAPHISTRY_PASSWORD\"] = \"\"\n", + "\n", + "if \"LLM_API_KEY\" not in os.environ:\n", + " os.environ[\"LLM_API_KEY\"] = \"\"\n", "\n", "os.environ[\"GRAPH_DATABASE_PROVIDER\"]=\"networkx\" # \"neo4j\" or \"networkx\"\n", "# Not needed if using networkx\n", From 243f0e3a7b96c53e4a77a341fbc11c89588bef32 Mon Sep 17 00:00:00 2001 From: Igor Ilic Date: Tue, 29 Oct 2024 13:49:53 +0100 Subject: [PATCH 2/8] test: Fix notebook test Add missing dollar sign to github action for testing notebook Test #COG-462 --- .github/workflows/test_notebook.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_notebook.yml b/.github/workflows/test_notebook.yml index 659714d94..ec2a09fa6 100644 --- a/.github/workflows/test_notebook.yml +++ b/.github/workflows/test_notebook.yml @@ -49,8 +49,8 @@ jobs: env: ENV: 'dev' LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }} - GRAPHISTRY_USERNAME: {{ secrets.GRAPHISTRY_USERNAME }} - GRAPHISTRY_PASSWORD: {{ secrets.GRAPHISTRY_PASSWORD }} + GRAPHISTRY_USERNAME: ${{ secrets.GRAPHISTRY_USERNAME }} + GRAPHISTRY_PASSWORD: ${{ secrets.GRAPHISTRY_PASSWORD }} run: jupyter nbconvert \ --to notebook \ --execute notebooks/cognee_demo.ipynb \ From ec0f4c61b17083dace881e9fe1c29471097ead3b Mon Sep 17 00:00:00 2001 From: Igor Ilic Date: Tue, 29 Oct 2024 13:58:52 +0100 Subject: [PATCH 3/8] test: Add installing of jupyter notebook to github action Install jupyter notebooks in github action Test #COG-462 --- .github/workflows/test_notebook.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_notebook.yml b/.github/workflows/test_notebook.yml index ec2a09fa6..101cfe33e 100644 --- a/.github/workflows/test_notebook.yml +++ b/.github/workflows/test_notebook.yml @@ -43,7 +43,9 @@ jobs: installer-parallel: true - name: Install dependencies - run: poetry install --no-interaction + run: | + poetry install --no-interaction + poetry add jupyter --no-interaction - name: Execute Jupyter Notebook env: From 5b56eb5356f37ad1a049645a349d4885ece564eb Mon Sep 17 00:00:00 2001 From: Igor Ilic Date: Tue, 29 Oct 2024 14:02:47 +0100 Subject: [PATCH 4/8] test: Add poetry run to notebook github action Run jupyter notebook through poetry environment Test #COG-462 --- .github/workflows/test_notebook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_notebook.yml b/.github/workflows/test_notebook.yml index 101cfe33e..36ed11f27 100644 --- a/.github/workflows/test_notebook.yml +++ b/.github/workflows/test_notebook.yml @@ -53,7 +53,7 @@ jobs: LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }} GRAPHISTRY_USERNAME: ${{ secrets.GRAPHISTRY_USERNAME }} GRAPHISTRY_PASSWORD: ${{ secrets.GRAPHISTRY_PASSWORD }} - run: jupyter nbconvert \ + run: poetry run jupyter nbconvert \ --to notebook \ --execute notebooks/cognee_demo.ipynb \ --output notebooks/executed_notebook.ipynb \ From dbbe8e57e433012aa8daccfd296579da3e450d61 Mon Sep 17 00:00:00 2001 From: Igor Ilic Date: Tue, 29 Oct 2024 14:11:41 +0100 Subject: [PATCH 5/8] test: Attempt to resolve newline issue Attempt at resolving issue of github actions understanding run command Test #COG-462 --- .github/workflows/test_notebook.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_notebook.yml b/.github/workflows/test_notebook.yml index 36ed11f27..54bf28ab9 100644 --- a/.github/workflows/test_notebook.yml +++ b/.github/workflows/test_notebook.yml @@ -53,8 +53,9 @@ jobs: LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }} GRAPHISTRY_USERNAME: ${{ secrets.GRAPHISTRY_USERNAME }} GRAPHISTRY_PASSWORD: ${{ secrets.GRAPHISTRY_PASSWORD }} - run: poetry run jupyter nbconvert \ - --to notebook \ - --execute notebooks/cognee_demo.ipynb \ - --output notebooks/executed_notebook.ipynb \ - --ExecutePreprocessor.timeout=1200 \ No newline at end of file + run: | + poetry run jupyter nbconvert \ + --to notebook \ + --execute notebooks/cognee_demo.ipynb \ + --output notebooks/executed_notebook.ipynb \ + --ExecutePreprocessor.timeout=1200 \ No newline at end of file From b09b3f169386a80b7ac52fc4a494bd0fdf842d60 Mon Sep 17 00:00:00 2001 From: Igor Ilic Date: Tue, 29 Oct 2024 14:16:08 +0100 Subject: [PATCH 6/8] test: Fix output path for notebook action Fix output path for jupyter notebook github action Fix #COG-462 --- .github/workflows/test_notebook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_notebook.yml b/.github/workflows/test_notebook.yml index 54bf28ab9..5e57ada02 100644 --- a/.github/workflows/test_notebook.yml +++ b/.github/workflows/test_notebook.yml @@ -57,5 +57,5 @@ jobs: poetry run jupyter nbconvert \ --to notebook \ --execute notebooks/cognee_demo.ipynb \ - --output notebooks/executed_notebook.ipynb \ + --output executed_notebook.ipynb \ --ExecutePreprocessor.timeout=1200 \ No newline at end of file From 2ba57220d8d93afe9f723d053cb5f61582db5085 Mon Sep 17 00:00:00 2001 From: Igor Ilic Date: Tue, 29 Oct 2024 14:20:47 +0100 Subject: [PATCH 7/8] test: Add typo in notebook to test github action Added typo in notebook to test if github action will catch the issue Test #COG-462 --- notebooks/cognee_demo.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/cognee_demo.ipynb b/notebooks/cognee_demo.ipynb index c2c249538..34497f9e4 100644 --- a/notebooks/cognee_demo.ipynb +++ b/notebooks/cognee_demo.ipynb @@ -658,7 +658,7 @@ " cognee_config = get_cognify_config()\n", "\n", " tasks = [\n", - " Task(classify_documents),\n", + " Task(classify_documentss),\n", " Task(check_permissions_on_documents, user = user, permissions = [\"write\"]),\n", " Task(infer_data_ontology, root_node_id = root_node_id, ontology_model = KnowledgeGraph),\n", " Task(source_documents_to_chunks, chunk_size = 800, parent_node_id = root_node_id), # Classify documents and save them as a nodes in graph db, extract text chunks based on the document type\n", From 23ed38d615902195319d6b82cd424ffd18b80ddf Mon Sep 17 00:00:00 2001 From: Igor Ilic Date: Tue, 29 Oct 2024 14:24:17 +0100 Subject: [PATCH 8/8] test: Fix intentional typo in notebook Removed typo used for testing notebook github action Test #COG-462 --- notebooks/cognee_demo.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/cognee_demo.ipynb b/notebooks/cognee_demo.ipynb index 34497f9e4..c2c249538 100644 --- a/notebooks/cognee_demo.ipynb +++ b/notebooks/cognee_demo.ipynb @@ -658,7 +658,7 @@ " cognee_config = get_cognify_config()\n", "\n", " tasks = [\n", - " Task(classify_documentss),\n", + " Task(classify_documents),\n", " Task(check_permissions_on_documents, user = user, permissions = [\"write\"]),\n", " Task(infer_data_ontology, root_node_id = root_node_id, ontology_model = KnowledgeGraph),\n", " Task(source_documents_to_chunks, chunk_size = 800, parent_node_id = root_node_id), # Classify documents and save them as a nodes in graph db, extract text chunks based on the document type\n",