faeca138d9
<!-- .github/pull_request_template.md --> ## Description This PR fixes distributed pipeline + updates core changes in distr logic. ## Type of Change <!-- Please check the relevant option --> - [x] Bug fix (non-breaking change that fixes an issue) - [x] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [x] Code refactoring - [x] Performance improvement - [ ] Other (please specify): ## Changes Made Fixes distributed pipeline: -Changed spawning logic + adds incremental loading to run_tasks_diistributed -Adds batching to consumer nodes -Fixes consumer stopping criteria by adding stop signal + handling -Changed edge embedding solution to avoid huge network load in a case of a multicontainer environment ## Testing Tested it by running 1GB on modal + manually ## Screenshots/Videos (if applicable) None ## Pre-submission Checklist <!-- Please check all boxes that apply before submitting your PR --> - [x] **I have tested my changes thoroughly before submitting this PR** - [x] **This PR contains minimal changes necessary to address the issue/feature** - [ ] My code follows the project's coding standards and style guidelines - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] I have added necessary documentation (if applicable) - [ ] All new and existing tests pass - [ ] I have searched existing PRs to ensure this change hasn't been submitted already - [ ] I have linked any relevant issues in the description - [ ] My commits have clear and descriptive messages ## Related Issues None ## Additional Notes None ## DCO Affirmation I affirm that all code in every commit of this pull request conforms to the terms of the Topoteretes Developer Certificate of Origin. --------- Co-authored-by: Boris <boris@topoteretes.com> Co-authored-by: Boris Arzentar <borisarzentar@gmail.com>
73 lines
2.4 KiB
YAML
73 lines
2.4 KiB
YAML
name: Distributed Cognee test with modal
|
|
permissions:
|
|
contents: read
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
python-version:
|
|
required: false
|
|
type: string
|
|
default: '3.11.x'
|
|
secrets:
|
|
LLM_MODEL:
|
|
required: true
|
|
LLM_ENDPOINT:
|
|
required: true
|
|
LLM_API_KEY:
|
|
required: true
|
|
LLM_API_VERSION:
|
|
required: true
|
|
EMBEDDING_MODEL:
|
|
required: true
|
|
EMBEDDING_ENDPOINT:
|
|
required: true
|
|
EMBEDDING_API_KEY:
|
|
required: true
|
|
EMBEDDING_API_VERSION:
|
|
required: true
|
|
OPENAI_API_KEY:
|
|
required: true
|
|
|
|
jobs:
|
|
run-server-start-test:
|
|
name: Distributed Cognee test (Modal)
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- name: Check out
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Cognee Setup
|
|
uses: ./.github/actions/cognee_setup
|
|
with:
|
|
python-version: '3.11.x'
|
|
extra-dependencies: "distributed postgres"
|
|
|
|
- name: Run Distributed Cognee (Modal)
|
|
env:
|
|
ENV: 'dev'
|
|
LLM_MODEL: ${{ secrets.LLM_MODEL }}
|
|
LLM_ENDPOINT: ${{ secrets.LLM_ENDPOINT }}
|
|
LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
|
|
LLM_API_VERSION: ${{ secrets.LLM_API_VERSION }}
|
|
EMBEDDING_MODEL: ${{ secrets.EMBEDDING_MODEL }}
|
|
EMBEDDING_ENDPOINT: ${{ secrets.EMBEDDING_ENDPOINT }}
|
|
EMBEDDING_API_KEY: ${{ secrets.EMBEDDING_API_KEY }}
|
|
EMBEDDING_API_VERSION: ${{ secrets.EMBEDDING_API_VERSION }}
|
|
MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
|
|
MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
|
|
MODAL_SECRET_NAME: ${{ secrets.MODAL_SECRET_NAME }}
|
|
GRAPH_DATABASE_PROVIDER: "neo4j"
|
|
GRAPH_DATABASE_URL: ${{ secrets.AZURE_NEO4j_URL }}
|
|
GRAPH_DATABASE_USERNAME: ${{ secrets.AZURE_NEO4J_USERNAME }}
|
|
GRAPH_DATABASE_PASSWORD: ${{ secrets.AZURE_NEO4J_PW }}
|
|
DB_PROVIDER: "postgres"
|
|
DB_NAME: ${{ secrets.AZURE_POSTGRES_DB_NAME }}
|
|
DB_HOST: ${{ secrets.AZURE_POSTGRES_HOST }}
|
|
DB_PORT: ${{ secrets.AZURE_POSTGRES_PORT }}
|
|
DB_USERNAME: ${{ secrets.AZURE_POSTGRES_USERNAME }}
|
|
DB_PASSWORD: ${{ secrets.AZURE_POSTGRES_PW }}
|
|
VECTOR_DB_PROVIDER: "pgvector"
|
|
COGNEE_DISTRIBUTED: "true"
|
|
run: uv run modal run ./distributed/entrypoint.py
|