From 39c210681a483e042208b9b840c8066bd982b24c Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:17:35 +0100 Subject: [PATCH 01/32] Added basic profiling --- .github/workflows/profiling.yaml | 70 ++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/profiling.yaml diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml new file mode 100644 index 000000000..543306aac --- /dev/null +++ b/.github/workflows/profiling.yaml @@ -0,0 +1,70 @@ +name: Profiling Comparison for Specific File + +on: + push +jobs: + profiler: + runs-on: ubuntu-latest + + steps: + # Checkout the code from the repository + - name: Checkout code + uses: actions/checkout@v3 + + # Set up Python environment + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + # Install necessary dependencies + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install scalene # Replace with your preferred profiler + pip install requests # For posting PR comments if needed + + # Run profiler on the base branch + - name: Run profiler on base branch + run: | + echo "Profiling the base branch for code_graph_pipeline.py" + git checkout ${{ github.event.pull_request.base.sha }} + scalene --profile-only cognee/api/v1/cognify/code_graph_pipeline.py --json-output base_results.json + + # Run profiler on the head branch + - name: Run profiler on head branch + run: | + echo "Profiling the head branch for code_graph_pipeline.py" + git checkout ${{ github.event.pull_request.head.sha }} + scalene --profile-only cognee/api/v1/cognify/code_graph_pipeline.py --json-output head_results.json + + # Compare profiling results + - name: Compare profiling results + id: compare + run: | + python -c " +import json +with open('base_results.json') as f: + base = json.load(f) +with open('head_results.json') as f: + head = json.load(f) +cpu_diff = head['total_cpu_samples'] - base['total_cpu_samples'] +memory_diff = head['total_memory_mb'] - base['total_memory_mb'] +with open('profiling_diff.txt', 'w') as f: + f.write(f'CPU Usage Difference: {cpu_diff}\\n') + f.write(f'Memory Usage Difference: {memory_diff:.2f} MB\\n') + " + + # Post results to the pull request + - name: Post profiling results to PR + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const diff = fs.readFileSync('profiling_diff.txt', 'utf-8'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `### Profiling Results for code_graph_pipeline.py\n\`\`\`\n${diff}\n\`\`\`` + }); From a935940d35c6a12be9b0b3996bb35b514408f783 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:18:26 +0100 Subject: [PATCH 02/32] Added basic profiling --- .github/workflows/profiling.yaml | 50 ++++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 543306aac..976a61f53 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -43,28 +43,28 @@ jobs: id: compare run: | python -c " -import json -with open('base_results.json') as f: - base = json.load(f) -with open('head_results.json') as f: - head = json.load(f) -cpu_diff = head['total_cpu_samples'] - base['total_cpu_samples'] -memory_diff = head['total_memory_mb'] - base['total_memory_mb'] -with open('profiling_diff.txt', 'w') as f: - f.write(f'CPU Usage Difference: {cpu_diff}\\n') - f.write(f'Memory Usage Difference: {memory_diff:.2f} MB\\n') - " - - # Post results to the pull request - - name: Post profiling results to PR - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const diff = fs.readFileSync('profiling_diff.txt', 'utf-8'); - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `### Profiling Results for code_graph_pipeline.py\n\`\`\`\n${diff}\n\`\`\`` - }); + import json + with open('base_results.json') as f: + base = json.load(f) + with open('head_results.json') as f: + head = json.load(f) + cpu_diff = head['total_cpu_samples'] - base['total_cpu_samples'] + memory_diff = head['total_memory_mb'] - base['total_memory_mb'] + with open('profiling_diff.txt', 'w') as f: + f.write(f'CPU Usage Difference: {cpu_diff}\\n') + f.write(f'Memory Usage Difference: {memory_diff:.2f} MB\\n') + " + + # Post results to the pull request + - name: Post profiling results to PR + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const diff = fs.readFileSync('profiling_diff.txt', 'utf-8'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `### Profiling Results for code_graph_pipeline.py\n\`\`\`\n${diff}\n\`\`\`` + }); From 7e66d5038e3573491f3b5f1faad196c0a9bf8019 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:20:02 +0100 Subject: [PATCH 03/32] Added basic profiling --- .github/workflows/profiling.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 976a61f53..c49f6ef87 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -29,14 +29,14 @@ jobs: run: | echo "Profiling the base branch for code_graph_pipeline.py" git checkout ${{ github.event.pull_request.base.sha }} - scalene --profile-only cognee/api/v1/cognify/code_graph_pipeline.py --json-output base_results.json + scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py --json-output base_results.json # Run profiler on the head branch - name: Run profiler on head branch run: | echo "Profiling the head branch for code_graph_pipeline.py" git checkout ${{ github.event.pull_request.head.sha }} - scalene --profile-only cognee/api/v1/cognify/code_graph_pipeline.py --json-output head_results.json + scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py --json-output head_results.json # Compare profiling results - name: Compare profiling results From df7bbfe3b9b82af4420505b6c2e16b3f4de51dcc Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:22:05 +0100 Subject: [PATCH 04/32] Added basic profiling --- .github/workflows/profiling.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index c49f6ef87..94af6e442 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -29,14 +29,14 @@ jobs: run: | echo "Profiling the base branch for code_graph_pipeline.py" git checkout ${{ github.event.pull_request.base.sha }} - scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py --json-output base_results.json + scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py --json-output=base_results.json # Run profiler on the head branch - name: Run profiler on head branch run: | echo "Profiling the head branch for code_graph_pipeline.py" git checkout ${{ github.event.pull_request.head.sha }} - scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py --json-output head_results.json + scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py --json-output=head_results.json # Compare profiling results - name: Compare profiling results From d589255b255e697906b9965892cac41478ea7695 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:24:14 +0100 Subject: [PATCH 05/32] Added basic profiling --- .github/workflows/profiling.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 94af6e442..5ad4b5664 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -29,14 +29,14 @@ jobs: run: | echo "Profiling the base branch for code_graph_pipeline.py" git checkout ${{ github.event.pull_request.base.sha }} - scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py --json-output=base_results.json + scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py # Run profiler on the head branch - name: Run profiler on head branch run: | echo "Profiling the head branch for code_graph_pipeline.py" git checkout ${{ github.event.pull_request.head.sha }} - scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py --json-output=head_results.json + scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py # Compare profiling results - name: Compare profiling results From 94688ed340eb9be5331ac2f080bb9b8952010fea Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:27:46 +0100 Subject: [PATCH 06/32] Added basic profiling --- .github/workflows/profiling.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 5ad4b5664..77215075a 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -29,6 +29,7 @@ jobs: run: | echo "Profiling the base branch for code_graph_pipeline.py" git checkout ${{ github.event.pull_request.base.sha }} + echo "This is the working directory: $PWD" scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py # Run profiler on the head branch @@ -36,6 +37,7 @@ jobs: run: | echo "Profiling the head branch for code_graph_pipeline.py" git checkout ${{ github.event.pull_request.head.sha }} + echo "This is the working directory: $PWD" scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py # Compare profiling results From 6171cd71caa160901d3048eb4b92c43e98f5ab5e Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:28:37 +0100 Subject: [PATCH 07/32] Added basic profiling --- .github/workflows/profiling.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 77215075a..332c29ef1 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -30,7 +30,7 @@ jobs: echo "Profiling the base branch for code_graph_pipeline.py" git checkout ${{ github.event.pull_request.base.sha }} echo "This is the working directory: $PWD" - scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py + scalene --profile-only /home/runner/work/cognee/cognee/api/v1/cognify/code_graph_pipeline.py # Run profiler on the head branch - name: Run profiler on head branch @@ -38,7 +38,7 @@ jobs: echo "Profiling the head branch for code_graph_pipeline.py" git checkout ${{ github.event.pull_request.head.sha }} echo "This is the working directory: $PWD" - scalene --profile-only $GITHUB_WORKSPACE/cognee/api/v1/cognify/code_graph_pipeline.py + scalene --profile-only /home/runner/work/cognee/cognee/api/v1/cognify/code_graph_pipeline.py # Compare profiling results - name: Compare profiling results From bba32aac51ad9b5252ac1156406a46276d328334 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:30:58 +0100 Subject: [PATCH 08/32] Added basic profiling --- .github/workflows/profiling.yaml | 97 ++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 35 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 332c29ef1..cdbaeacc7 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -1,15 +1,20 @@ name: Profiling Comparison for Specific File on: - push + pull_request: + paths: + - 'cognee/api/v1/cognify/code_graph_pipeline.py' # Trigger only if this specific file changes + jobs: profiler: runs-on: ubuntu-latest steps: - # Checkout the code from the repository + # Checkout the code from the repository with full history - name: Checkout code uses: actions/checkout@v3 + with: + fetch-depth: 0 # Fetch all history so we can checkout any commit # Set up Python environment - name: Set up Python @@ -21,52 +26,74 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install scalene # Replace with your preferred profiler - pip install requests # For posting PR comments if needed + pip install scalene requests + # Install your project's dependencies if needed + pip install -r requirements.txt # If you have a requirements file + + # Set environment variables for SHAs + - name: Set environment variables + run: | + echo "BASE_SHA=${{ github.event.pull_request.base.sha }}" >> $GITHUB_ENV + echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV # Run profiler on the base branch - name: Run profiler on base branch + env: + BASE_SHA: ${{ env.BASE_SHA }} run: | echo "Profiling the base branch for code_graph_pipeline.py" - git checkout ${{ github.event.pull_request.base.sha }} + echo "Checking out base SHA: $BASE_SHA" + git checkout $BASE_SHA echo "This is the working directory: $PWD" - scalene --profile-only /home/runner/work/cognee/cognee/api/v1/cognify/code_graph_pipeline.py + # Ensure the script is executable + chmod +x cognee/api/v1/cognify/code_graph_pipeline.py + # Run Scalene + scalene --json --outfile base_results.json cognee/api/v1/cognify/code_graph_pipeline.py - # Run profiler on the head branch + # Run profiler on head branch - name: Run profiler on head branch + env: + HEAD_SHA: ${{ env.HEAD_SHA }} run: | echo "Profiling the head branch for code_graph_pipeline.py" - git checkout ${{ github.event.pull_request.head.sha }} + echo "Checking out head SHA: $HEAD_SHA" + git checkout $HEAD_SHA echo "This is the working directory: $PWD" - scalene --profile-only /home/runner/work/cognee/cognee/api/v1/cognify/code_graph_pipeline.py + # Ensure the script is executable + chmod +x cognee/api/v1/cognify/code_graph_pipeline.py + # Run Scalene + scalene --json --outfile head_results.json cognee/api/v1/cognify/code_graph_pipeline.py # Compare profiling results - name: Compare profiling results - id: compare run: | python -c " - import json - with open('base_results.json') as f: - base = json.load(f) - with open('head_results.json') as f: - head = json.load(f) - cpu_diff = head['total_cpu_samples'] - base['total_cpu_samples'] - memory_diff = head['total_memory_mb'] - base['total_memory_mb'] - with open('profiling_diff.txt', 'w') as f: - f.write(f'CPU Usage Difference: {cpu_diff}\\n') - f.write(f'Memory Usage Difference: {memory_diff:.2f} MB\\n') - " - - # Post results to the pull request - - name: Post profiling results to PR - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const diff = fs.readFileSync('profiling_diff.txt', 'utf-8'); - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `### Profiling Results for code_graph_pipeline.py\n\`\`\`\n${diff}\n\`\`\`` - }); +import json +try: + with open('base_results.json') as f: + base = json.load(f) + with open('head_results.json') as f: + head = json.load(f) + cpu_diff = head.get('total_cpu_samples_python', 0) - base.get('total_cpu_samples_python', 0) + memory_diff = head.get('malloc_samples', 0) - base.get('malloc_samples', 0) + with open('profiling_diff.txt', 'w') as f: + f.write(f'CPU Usage Difference: {cpu_diff}\\n') + f.write(f'Memory Usage Difference: {memory_diff} bytes\\n') +except Exception as e: + with open('profiling_diff.txt', 'w') as f: + f.write(f'Error comparing profiling results: {e}\\n') +" + + # Post results to the pull request + - name: Post profiling results to PR + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const diff = fs.readFileSync('profiling_diff.txt', 'utf-8'); + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `### Profiling Results for code_graph_pipeline.py\n\`\`\`\n${diff || 'No differences found.'}\n\`\`\`` + }); From a904b8d1fc4529d1dc74090f049c261c8dd25a01 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:31:49 +0100 Subject: [PATCH 09/32] Added basic profiling --- .github/workflows/profiling.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index cdbaeacc7..267755461 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -68,21 +68,21 @@ jobs: - name: Compare profiling results run: | python -c " -import json -try: - with open('base_results.json') as f: - base = json.load(f) - with open('head_results.json') as f: - head = json.load(f) - cpu_diff = head.get('total_cpu_samples_python', 0) - base.get('total_cpu_samples_python', 0) - memory_diff = head.get('malloc_samples', 0) - base.get('malloc_samples', 0) - with open('profiling_diff.txt', 'w') as f: - f.write(f'CPU Usage Difference: {cpu_diff}\\n') - f.write(f'Memory Usage Difference: {memory_diff} bytes\\n') -except Exception as e: - with open('profiling_diff.txt', 'w') as f: - f.write(f'Error comparing profiling results: {e}\\n') -" + import json + try: + with open('base_results.json') as f: + base = json.load(f) + with open('head_results.json') as f: + head = json.load(f) + cpu_diff = head.get('total_cpu_samples_python', 0) - base.get('total_cpu_samples_python', 0) + memory_diff = head.get('malloc_samples', 0) - base.get('malloc_samples', 0) + with open('profiling_diff.txt', 'w') as f: + f.write(f'CPU Usage Difference: {cpu_diff}\\n') + f.write(f'Memory Usage Difference: {memory_diff} bytes\\n') + except Exception as e: + with open('profiling_diff.txt', 'w') as f: + f.write(f'Error comparing profiling results: {e}\\n') + " # Post results to the pull request - name: Post profiling results to PR From bdef152dac231f07f7fe4434adb33f141c8814e2 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:33:05 +0100 Subject: [PATCH 10/32] Added basic profiling --- .github/workflows/profiling.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 267755461..1031d7bc0 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -1,4 +1,4 @@ -name: Profiling Comparison for Specific File +name: Profiling Comparison for Specific File 2 on: pull_request: From e2539cd396df8a5311dff7e1f66896ff9ccb1be7 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:35:06 +0100 Subject: [PATCH 11/32] Added basic profiling --- .github/workflows/profiling.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 1031d7bc0..7355dc74f 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -66,23 +66,23 @@ jobs: # Compare profiling results - name: Compare profiling results - run: | + run: | python -c " - import json - try: - with open('base_results.json') as f: - base = json.load(f) - with open('head_results.json') as f: - head = json.load(f) - cpu_diff = head.get('total_cpu_samples_python', 0) - base.get('total_cpu_samples_python', 0) - memory_diff = head.get('malloc_samples', 0) - base.get('malloc_samples', 0) - with open('profiling_diff.txt', 'w') as f: - f.write(f'CPU Usage Difference: {cpu_diff}\\n') - f.write(f'Memory Usage Difference: {memory_diff} bytes\\n') - except Exception as e: - with open('profiling_diff.txt', 'w') as f: - f.write(f'Error comparing profiling results: {e}\\n') - " + import json + try: + with open('base_results.json') as f: + base = json.load(f) + with open('head_results.json') as f: + head = json.load(f) + cpu_diff = head.get('total_cpu_samples_python', 0) - base.get('total_cpu_samples_python', 0) + memory_diff = head.get('malloc_samples', 0) - base.get('malloc_samples', 0) + with open('profiling_diff.txt', 'w') as f: + f.write(f'CPU Usage Difference: {cpu_diff}\\n') + f.write(f'Memory Usage Difference: {memory_diff} bytes\\n') + except Exception as e: + with open('profiling_diff.txt', 'w') as f: + f.write(f'Error comparing profiling results: {e}\\n') + " # Post results to the pull request - name: Post profiling results to PR From 6ab427e0ad7cbece2705a23bc4042ef772faaad4 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:38:07 +0100 Subject: [PATCH 12/32] Added basic profiling --- .github/workflows/profiling.yaml | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 7355dc74f..a5bb0ee0b 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -66,23 +66,23 @@ jobs: # Compare profiling results - name: Compare profiling results - run: | - python -c " - import json - try: - with open('base_results.json') as f: - base = json.load(f) - with open('head_results.json') as f: - head = json.load(f) - cpu_diff = head.get('total_cpu_samples_python', 0) - base.get('total_cpu_samples_python', 0) - memory_diff = head.get('malloc_samples', 0) - base.get('malloc_samples', 0) - with open('profiling_diff.txt', 'w') as f: - f.write(f'CPU Usage Difference: {cpu_diff}\\n') - f.write(f'Memory Usage Difference: {memory_diff} bytes\\n') - except Exception as e: - with open('profiling_diff.txt', 'w') as f: - f.write(f'Error comparing profiling results: {e}\\n') - " + run: | + python -c ' + import json + try: + with open("base_results.json") as f: + base = json.load(f) + with open("head_results.json") as f: + head = json.load(f) + cpu_diff = head.get("total_cpu_samples_python", 0) - base.get("total_cpu_samples_python", 0) + memory_diff = head.get("malloc_samples", 0) - base.get("malloc_samples", 0) + with open("profiling_diff.txt", "w") as f: + f.write(f"CPU Usage Difference: {cpu_diff}\\n") + f.write(f"Memory Usage Difference: {memory_diff} bytes\\n") + except Exception as e: + with open("profiling_diff.txt", "w") as f: + f.write(f"Error comparing profiling results: {e}\\n") + ' # Post results to the pull request - name: Post profiling results to PR From fa608270bb6952b0ae23a87dcb7aba8cb3763084 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:40:07 +0100 Subject: [PATCH 13/32] Added basic profiling --- .github/workflows/profiling.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index a5bb0ee0b..3e1054cf6 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -1,10 +1,7 @@ name: Profiling Comparison for Specific File 2 on: - pull_request: - paths: - - 'cognee/api/v1/cognify/code_graph_pipeline.py' # Trigger only if this specific file changes - + push jobs: profiler: runs-on: ubuntu-latest From 32ca751abecdad7dd40321d0c6a24938a563619f Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:40:56 +0100 Subject: [PATCH 14/32] Added basic profiling --- .github/workflows/profiling.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 3e1054cf6..0a9bfa97a 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -24,8 +24,7 @@ jobs: run: | python -m pip install --upgrade pip pip install scalene requests - # Install your project's dependencies if needed - pip install -r requirements.txt # If you have a requirements file + # Set environment variables for SHAs - name: Set environment variables From d523f71b79d5d60882b0381e557be4933036cb90 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:42:26 +0100 Subject: [PATCH 15/32] Added basic profiling --- .github/workflows/profiling.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 0a9bfa97a..15ad96303 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -17,7 +17,18 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.11' + + - name: Install Poetry + uses: snok/install-poetry@v1.3.2 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Install dependencies + run: | + poetry install --no-interaction --all-extras # Install necessary dependencies - name: Install dependencies From c2896f3b5b29295ae25c9ad3323e154757edf84e Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:45:54 +0100 Subject: [PATCH 16/32] Added basic profiling --- .github/workflows/profiling.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 15ad96303..a6214ebff 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -29,12 +29,7 @@ jobs: - name: Install dependencies run: | poetry install --no-interaction --all-extras - - # Install necessary dependencies - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install scalene requests + poetry add scalene requests # Set environment variables for SHAs @@ -55,7 +50,7 @@ jobs: # Ensure the script is executable chmod +x cognee/api/v1/cognify/code_graph_pipeline.py # Run Scalene - scalene --json --outfile base_results.json cognee/api/v1/cognify/code_graph_pipeline.py + poetry run scalene --json --outfile base_results.json cognee/api/v1/cognify/code_graph_pipeline.py # Run profiler on head branch - name: Run profiler on head branch @@ -69,7 +64,7 @@ jobs: # Ensure the script is executable chmod +x cognee/api/v1/cognify/code_graph_pipeline.py # Run Scalene - scalene --json --outfile head_results.json cognee/api/v1/cognify/code_graph_pipeline.py + poetry run scalene --json --outfile head_results.json cognee/api/v1/cognify/code_graph_pipeline.py # Compare profiling results - name: Compare profiling results From e178d38ec6b8c8cd131a83d4ab88b4e4f4dcb892 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 18:48:34 +0100 Subject: [PATCH 17/32] Added basic profiling --- .github/workflows/profiling.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index a6214ebff..dfba89270 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.10' - name: Install Poetry uses: snok/install-poetry@v1.3.2 From 21c7b8eee9d89f5091a4fc2c02ec0b298399dc11 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:20:48 +0100 Subject: [PATCH 18/32] Bump release version --- .github/workflows/profiling.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index dfba89270..e1ac05a9d 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -29,7 +29,7 @@ jobs: - name: Install dependencies run: | poetry install --no-interaction --all-extras - poetry add scalene requests + poetry add scalene # Set environment variables for SHAs From 54b88448c52bcd941d6965c3ff340b7c6bdc8b96 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:24:20 +0100 Subject: [PATCH 19/32] Bump release version --- .github/workflows/profiling.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index e1ac05a9d..175d824a7 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -29,7 +29,7 @@ jobs: - name: Install dependencies run: | poetry install --no-interaction --all-extras - poetry add scalene + poetry add pyinstrument # Set environment variables for SHAs @@ -50,7 +50,7 @@ jobs: # Ensure the script is executable chmod +x cognee/api/v1/cognify/code_graph_pipeline.py # Run Scalene - poetry run scalene --json --outfile base_results.json cognee/api/v1/cognify/code_graph_pipeline.py + poetry run pyinstrument --json --outfile base_results.json cognee/api/v1/cognify/code_graph_pipeline.py # Run profiler on head branch - name: Run profiler on head branch @@ -64,7 +64,7 @@ jobs: # Ensure the script is executable chmod +x cognee/api/v1/cognify/code_graph_pipeline.py # Run Scalene - poetry run scalene --json --outfile head_results.json cognee/api/v1/cognify/code_graph_pipeline.py + poetry run pyinstrument --json --outfile head_results.json cognee/api/v1/cognify/code_graph_pipeline.py # Compare profiling results - name: Compare profiling results From 692770c197030e4ad2dfb934e77b2f28595d3825 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:28:21 +0100 Subject: [PATCH 20/32] Bump release version --- .github/workflows/profiling.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 175d824a7..aa4595a57 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -29,7 +29,7 @@ jobs: - name: Install dependencies run: | poetry install --no-interaction --all-extras - poetry add pyinstrument + poetry run pip install pyinstrument # Set environment variables for SHAs From 2df1eb6098b5b979507a829f2544a4b11b569f05 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:31:14 +0100 Subject: [PATCH 21/32] Bump release version --- .github/workflows/profiling.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index aa4595a57..3f2cb6283 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -50,7 +50,7 @@ jobs: # Ensure the script is executable chmod +x cognee/api/v1/cognify/code_graph_pipeline.py # Run Scalene - poetry run pyinstrument --json --outfile base_results.json cognee/api/v1/cognify/code_graph_pipeline.py + poetry run pyinstrument --renderer json -o base_results.json cognee/api/v1/cognify/code_graph_pipeline.py # Run profiler on head branch - name: Run profiler on head branch @@ -64,7 +64,7 @@ jobs: # Ensure the script is executable chmod +x cognee/api/v1/cognify/code_graph_pipeline.py # Run Scalene - poetry run pyinstrument --json --outfile head_results.json cognee/api/v1/cognify/code_graph_pipeline.py + poetry run pyinstrument --renderer json -o head_results.json cognee/api/v1/cognify/code_graph_pipeline.py # Compare profiling results - name: Compare profiling results From 8d1936f022675ea8fe9d041e63b8fe4de543ed10 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:35:06 +0100 Subject: [PATCH 22/32] Bump release version --- .github/workflows/profiling.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 3f2cb6283..d77305f1d 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -1,7 +1,7 @@ name: Profiling Comparison for Specific File 2 on: - push + pull_request jobs: profiler: runs-on: ubuntu-latest From f37d96df6eed585fc3f3c7c1a870b242a40d618f Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:44:34 +0100 Subject: [PATCH 23/32] Bump release version --- .github/workflows/profiling.yaml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index d77305f1d..c64c32aaf 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -1,7 +1,7 @@ name: Profiling Comparison for Specific File 2 on: - pull_request + pull_request_target jobs: profiler: runs-on: ubuntu-latest @@ -13,6 +13,30 @@ jobs: with: fetch-depth: 0 # Fetch all history so we can checkout any commit + - name: Check if the sender is a maintainer + id: check_permissions + uses: actions/github-script@v6 + with: + script: | + const sender = context.payload.sender.login; + const { data: membership } = await github.rest.orgs.getMembershipForUser({ + org: context.repo.owner, + username: sender, + }).catch(() => ({ data: { role: null } })); + return membership.role; + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set is_maintainer variable + run: | + echo "is_maintainer=${{ steps.check_permissions.outputs.result == 'admin' || steps.check_permissions.outputs.result == 'maintainer' }}" >> $GITHUB_ENV + + - name: Stop if not a maintainer + if: env.is_maintainer != 'true' + run: | + echo "User ${{ github.event.sender.login }} is not a maintainer. Exiting." + exit 0 # Use exit 0 to mark the job as successful but stop execution + # Set up Python environment - name: Set up Python uses: actions/setup-python@v4 From cc43a8c86598fe9f26082a3e9845ce8ca93dc21c Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:48:01 +0100 Subject: [PATCH 24/32] Bump release version --- .github/workflows/profiling.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index c64c32aaf..c3c4d460b 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -1,7 +1,11 @@ name: Profiling Comparison for Specific File 2 on: - pull_request_target + pull_request_target: + types: + - opened + - reopened + - synchronize jobs: profiler: runs-on: ubuntu-latest From cf515559439fc65714e90992f7dbafc68297ec26 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:50:25 +0100 Subject: [PATCH 25/32] Bump release version --- .github/workflows/profiling.yaml | 45 +++++++++++++++++--------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index c3c4d460b..ea73159d0 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -1,11 +1,7 @@ name: Profiling Comparison for Specific File 2 on: - pull_request_target: - types: - - opened - - reopened - - synchronize + push jobs: profiler: runs-on: ubuntu-latest @@ -97,6 +93,7 @@ jobs: # Compare profiling results - name: Compare profiling results run: | + | python -c ' import json try: @@ -106,24 +103,30 @@ jobs: head = json.load(f) cpu_diff = head.get("total_cpu_samples_python", 0) - base.get("total_cpu_samples_python", 0) memory_diff = head.get("malloc_samples", 0) - base.get("malloc_samples", 0) + results = [ + f"CPU Usage Difference: {cpu_diff}", + f"Memory Usage Difference: {memory_diff} bytes" + ] with open("profiling_diff.txt", "w") as f: - f.write(f"CPU Usage Difference: {cpu_diff}\\n") - f.write(f"Memory Usage Difference: {memory_diff} bytes\\n") + f.write("\\n".join(results) + "\\n") + print("\\n".join(results)) # Print results to terminal except Exception as e: + error_message = f"Error comparing profiling results: {e}" with open("profiling_diff.txt", "w") as f: - f.write(f"Error comparing profiling results: {e}\\n") - ' + f.write(error_message + "\\n") + print(error_message) # Print error to terminal + ' # Post results to the pull request - - name: Post profiling results to PR - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const diff = fs.readFileSync('profiling_diff.txt', 'utf-8'); - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `### Profiling Results for code_graph_pipeline.py\n\`\`\`\n${diff || 'No differences found.'}\n\`\`\`` - }); +# - name: Post profiling results to PR +# uses: actions/github-script@v6 +# with: +# script: | +# const fs = require('fs'); +# const diff = fs.readFileSync('profiling_diff.txt', 'utf-8'); +# github.rest.issues.createComment({ +# issue_number: context.issue.number, +# owner: context.repo.owner, +# repo: context.repo.repo, +# body: `### Profiling Results for code_graph_pipeline.py\n\`\`\`\n${diff || 'No differences found.'}\n\`\`\`` +# }); From d2fccc1315b5b6f62bc47fee11c644a4588fc297 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:53:33 +0100 Subject: [PATCH 26/32] Bump release version --- .github/workflows/profiling.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index ea73159d0..8058e9dca 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -93,7 +93,6 @@ jobs: # Compare profiling results - name: Compare profiling results run: | - | python -c ' import json try: From a96daef46928a3106115045ebd36d9e1af709467 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:56:09 +0100 Subject: [PATCH 27/32] Bump release version --- .github/workflows/profiling.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 8058e9dca..2c14e4a93 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -1,4 +1,4 @@ -name: Profiling Comparison for Specific File 2 +name: test | profiling on: push From c431e7cc61df7855c6d7576f00e48a8172a93e3c Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Tue, 10 Dec 2024 13:44:08 +0100 Subject: [PATCH 28/32] Update profiling.yaml --- .github/workflows/profiling.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 2c14e4a93..0ae7dfb0d 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -9,7 +9,7 @@ jobs: steps: # Checkout the code from the repository with full history - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 # Fetch all history so we can checkout any commit From 5609bbccc8b2088d0389a7edeec7df75835c04d5 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:12:59 +0100 Subject: [PATCH 29/32] removed issues --- .github/workflows/get_docs_changes.yml | 5 ----- .github/workflows/mkdocs.yml | 6 ------ .github/workflows/py_lint.yml | 6 ------ .github/workflows/reusable_notebook.yml | 5 ----- .github/workflows/reusable_python_example.yml | 5 ----- .github/workflows/test_milvus.yml | 5 ----- .github/workflows/test_neo4j.yml | 2 -- .github/workflows/test_pgvector.yml | 5 ----- .github/workflows/test_python_3_10.yml | 5 ----- .github/workflows/test_python_3_11.yml | 5 ----- .github/workflows/test_python_3_9.yml | 5 ----- .github/workflows/test_qdrant.yml | 5 ----- .github/workflows/test_weaviate.yml | 5 ----- 13 files changed, 64 deletions(-) diff --git a/.github/workflows/get_docs_changes.yml b/.github/workflows/get_docs_changes.yml index 5ac194d9e..4059f588d 100644 --- a/.github/workflows/get_docs_changes.yml +++ b/.github/workflows/get_docs_changes.yml @@ -12,11 +12,6 @@ env: jobs: - get_docs_changes: - name: docs changes - runs-on: ubuntu-latest - outputs: - changes_outside_docs: ${{ steps.check_changes.outputs.changes_outside_docs }} steps: - name: Checkout code diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml index 08a123c6b..bb3551e03 100644 --- a/.github/workflows/mkdocs.yml +++ b/.github/workflows/mkdocs.yml @@ -17,14 +17,8 @@ permissions: contents: write jobs: - get_docs_changes: - name: run docs changes - uses: ./.github/workflows/get_docs_changes.yml deploy: runs-on: ubuntu-latest - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'false' - steps: - name: Checkout code uses: actions/checkout@master diff --git a/.github/workflows/py_lint.yml b/.github/workflows/py_lint.yml index 761e0ff5d..2ed49c054 100644 --- a/.github/workflows/py_lint.yml +++ b/.github/workflows/py_lint.yml @@ -13,14 +13,8 @@ concurrency: cancel-in-progress: true jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml - run_lint: name: lint - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' strategy: fail-fast: true matrix: diff --git a/.github/workflows/reusable_notebook.yml b/.github/workflows/reusable_notebook.yml index e4bcfc14f..0c0b63ec4 100644 --- a/.github/workflows/reusable_notebook.yml +++ b/.github/workflows/reusable_notebook.yml @@ -19,14 +19,9 @@ env: RUNTIME__LOG_LEVEL: ERROR jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml run_notebook_test: name: test - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' && ${{ github.event.label.name == 'run-checks' }} runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/reusable_python_example.yml b/.github/workflows/reusable_python_example.yml index 5a8e47e64..03f928656 100644 --- a/.github/workflows/reusable_python_example.yml +++ b/.github/workflows/reusable_python_example.yml @@ -19,14 +19,9 @@ env: RUNTIME__LOG_LEVEL: ERROR jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml run_notebook_test: name: test - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' && ${{ github.event.label.name == 'run-checks' }} runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/test_milvus.yml b/.github/workflows/test_milvus.yml index d7f401632..5cad72378 100644 --- a/.github/workflows/test_milvus.yml +++ b/.github/workflows/test_milvus.yml @@ -14,14 +14,9 @@ env: ENV: 'dev' jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml run_milvus: name: test - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' && ${{ github.event.label.name == 'run-checks' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/test_neo4j.yml b/.github/workflows/test_neo4j.yml index 934ac653b..e96528134 100644 --- a/.github/workflows/test_neo4j.yml +++ b/.github/workflows/test_neo4j.yml @@ -19,8 +19,6 @@ jobs: run_neo4j_integration_test: name: test - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' && ${{ github.event.label.name == 'run-checks' }} runs-on: ubuntu-latest defaults: diff --git a/.github/workflows/test_pgvector.yml b/.github/workflows/test_pgvector.yml index bf4db6f9a..a162d2cb4 100644 --- a/.github/workflows/test_pgvector.yml +++ b/.github/workflows/test_pgvector.yml @@ -14,14 +14,9 @@ env: RUNTIME__LOG_LEVEL: ERROR jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml run_pgvector_integration_test: name: test - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' && ${{ github.event.label.name == 'run-checks' }} runs-on: ubuntu-latest defaults: run: diff --git a/.github/workflows/test_python_3_10.yml b/.github/workflows/test_python_3_10.yml index 30d266379..39eb4e57a 100644 --- a/.github/workflows/test_python_3_10.yml +++ b/.github/workflows/test_python_3_10.yml @@ -14,14 +14,9 @@ env: ENV: 'dev' jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml run_common: name: test - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' && ${{ github.event.label.name == 'run-checks' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/test_python_3_11.yml b/.github/workflows/test_python_3_11.yml index 1bf8b50f0..2dd704eb9 100644 --- a/.github/workflows/test_python_3_11.yml +++ b/.github/workflows/test_python_3_11.yml @@ -14,14 +14,9 @@ env: ENV: 'dev' jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml run_common: name: test - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' && ${{ github.event.label.name == 'run-checks' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/test_python_3_9.yml b/.github/workflows/test_python_3_9.yml index cac4d0cd5..99c2b9a7a 100644 --- a/.github/workflows/test_python_3_9.yml +++ b/.github/workflows/test_python_3_9.yml @@ -14,14 +14,9 @@ env: ENV: 'dev' jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml run_common: name: test - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' && ${{ github.event.label.name == 'run-checks' }} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/test_qdrant.yml b/.github/workflows/test_qdrant.yml index 4eaf03504..f0a2e3d3f 100644 --- a/.github/workflows/test_qdrant.yml +++ b/.github/workflows/test_qdrant.yml @@ -14,14 +14,9 @@ env: RUNTIME__LOG_LEVEL: ERROR jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml run_qdrant_integration_test: name: test - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' && ${{ github.event.label.name == 'run-checks' }} runs-on: ubuntu-latest defaults: diff --git a/.github/workflows/test_weaviate.yml b/.github/workflows/test_weaviate.yml index 711bd392f..b8eb72383 100644 --- a/.github/workflows/test_weaviate.yml +++ b/.github/workflows/test_weaviate.yml @@ -14,14 +14,9 @@ env: RUNTIME__LOG_LEVEL: ERROR jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml run_weaviate_integration_test: name: test - needs: get_docs_changes - if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' && ${{ github.event.label.name == 'run-checks' }} runs-on: ubuntu-latest defaults: From 0e680197162a8183a6833964813f13a7aea506bd Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:16:32 +0100 Subject: [PATCH 30/32] fix --- .github/workflows/get_docs_changes.yml | 44 ----------------------- .github/workflows/mkdocs.yml | 50 -------------------------- 2 files changed, 94 deletions(-) delete mode 100644 .github/workflows/get_docs_changes.yml delete mode 100644 .github/workflows/mkdocs.yml diff --git a/.github/workflows/get_docs_changes.yml b/.github/workflows/get_docs_changes.yml deleted file mode 100644 index 4059f588d..000000000 --- a/.github/workflows/get_docs_changes.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: util | get docs changes - -on: - workflow_call: - outputs: - changes_outside_docs: - description: "Changes outside docs" - value: ${{ jobs.get_docs_changes.outputs.changes_outside_docs }} - -env: - EXCLUDED_FILE_PATTERNS: '^docs/|^README.md|^CONTRIBUTING.md|^LICENSE\.txt|\.editorconfig|\.gitignore|get_docs_changes.yml' - - -jobs: - - steps: - - name: Checkout code - uses: actions/checkout@master - with: - fetch-depth: 0 - - - name: Check changes outside docs - id: check_changes - run: | - echo "base.sha: ${{ github.event.pull_request.base.sha }}" - echo "head.sha: ${{ github.event.pull_request.head.sha }}" - - merge_base_sha=$(git merge-base ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }}) - - echo "merge_base_sha: $merge_base_sha" - - changes_outside_docs=$( - git diff --name-only $merge_base_sha ${{ github.event.pull_request.head.sha }} \ - | grep -vE "${{ env.EXCLUDED_FILE_PATTERNS }}" || true - ) - - echo $changes_outside_docs - if [ -z "$changes_outside_docs" ]; then - echo "No changes outside docs. Skipping tests." - echo "changes_outside_docs=false" >> $GITHUB_OUTPUT - else - echo "Changes detected outside docs." - echo "changes_outside_docs=true" >> $GITHUB_OUTPUT - fi \ No newline at end of file diff --git a/.github/workflows/mkdocs.yml b/.github/workflows/mkdocs.yml deleted file mode 100644 index bb3551e03..000000000 --- a/.github/workflows/mkdocs.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: util docs | build and deploy - -on: - pull_request: - branches: - - main - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - RUNTIME__LOG_LEVEL: ERROR - -permissions: - contents: write - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@master - - - name: Install Poetry - uses: snok/install-poetry@v1.3.1 - - - name: Use output - run: echo "The stage is finished" - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.11.x' - - - name: Install APT packages - run: | - sudo apt-get update && - sudo apt-get install pngquant - - - name: Install via Poetry - run: poetry install --with dev,docs - env: - GH_TOKEN: ${{ secrets.PAT_FOR_CROSS_REPOS_CICD_TRIGGERING }} - - - name: Build and deploy MkDocs - run: poetry run mkdocs gh-deploy --force - env: - DOCS_SEGMENT_KEY: ${{ secrets.DOCS_SEGMENT_KEY }} From 7cc5607740e4a442c8f9e3c6952b25ae1a1671c6 Mon Sep 17 00:00:00 2001 From: Vasilije <8619304+Vasilije1990@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:30:09 +0100 Subject: [PATCH 31/32] fix --- .github/workflows/profiling.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml index 0ae7dfb0d..6ca09f9dc 100644 --- a/.github/workflows/profiling.yaml +++ b/.github/workflows/profiling.yaml @@ -116,6 +116,12 @@ jobs: print(error_message) # Print error to terminal ' + - name: Upload profiling diff artifact + uses: actions/upload-artifact@v3 + with: + name: profiling-diff + path: profiling_diff.txt + # Post results to the pull request # - name: Post profiling results to PR # uses: actions/github-script@v6 From 9a9ea751bff4fd5fdcf6d83a0c2353ed28d89b3c Mon Sep 17 00:00:00 2001 From: vasilije Date: Wed, 11 Dec 2024 13:37:48 +0100 Subject: [PATCH 32/32] Fix neo4j --- .github/workflows/test_neo4j.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test_neo4j.yml b/.github/workflows/test_neo4j.yml index e96528134..3f3a35e4f 100644 --- a/.github/workflows/test_neo4j.yml +++ b/.github/workflows/test_neo4j.yml @@ -13,10 +13,6 @@ env: RUNTIME__LOG_LEVEL: ERROR jobs: - get_docs_changes: - name: docs changes - uses: ./.github/workflows/get_docs_changes.yml - run_neo4j_integration_test: name: test runs-on: ubuntu-latest