From a16100f8cdc0c55810afe8939af0ce5932481b05 Mon Sep 17 00:00:00 2001 From: Andrej Milicevic Date: Thu, 11 Sep 2025 17:50:27 +0200 Subject: [PATCH 1/6] test: Add CI test for fastembed --- .../disable_independent_workflows.sh | 1 + .github/workflows/test_fastembed.yml | 28 +++++++++++++++++++ .github/workflows/test_suites.yml | 9 ++++++ 3 files changed, 38 insertions(+) create mode 100644 .github/workflows/test_fastembed.yml diff --git a/.github/workflows/disable_independent_workflows.sh b/.github/workflows/disable_independent_workflows.sh index 693c3092d..9ec4a711e 100755 --- a/.github/workflows/disable_independent_workflows.sh +++ b/.github/workflows/disable_independent_workflows.sh @@ -11,6 +11,7 @@ WORKFLOWS=( "test_multimetric_qa_eval_run.yaml" "test_graphrag_vs_rag_notebook.yml" "test_gemini.yml" + "test_fastembed.yml" "test_multimedia_example.yaml" "test_deduplication.yml" "test_eval_framework.yml" diff --git a/.github/workflows/test_fastembed.yml b/.github/workflows/test_fastembed.yml new file mode 100644 index 000000000..5d37c1118 --- /dev/null +++ b/.github/workflows/test_fastembed.yml @@ -0,0 +1,28 @@ +name: test | fastembed + +on: + workflow_call: + +jobs: + test-gemini: + name: Run Fastembed Test + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Cognee Setup + uses: ./.github/actions/cognee_setup + with: + python-version: '3.11.x' + + - name: Run Fastembed Simple Example + env: + LLM_PROVIDER: "openai" + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + LLM_MODEL: "gpt-4o-mini" + EMBEDDING_PROVIDER: "fastembed" + EMBEDDING_MODEL: "sentence-transformers/all-MiniLM-L6-v2" + EMBEDDING_DIMENSIONS: "384" + EMBEDDING_MAX_TOKENS: "256" + run: uv run python ./examples/python/simple_example.py diff --git a/.github/workflows/test_suites.yml b/.github/workflows/test_suites.yml index b6e84ac1e..06e3e10da 100644 --- a/.github/workflows/test_suites.yml +++ b/.github/workflows/test_suites.yml @@ -121,6 +121,12 @@ jobs: uses: ./.github/workflows/test_gemini.yml secrets: inherit + fastembed-tests: + name: Fastembed Tests + needs: [basic-tests, e2e-tests, cli-tests] + uses: ./.github/workflows/test_fastembed.yml + secrets: inherit + openrouter-tests: name: OpenRouter Tests needs: [basic-tests, e2e-tests, cli-tests] @@ -140,6 +146,7 @@ jobs: vector-db-tests, example-tests, gemini-tests, + fastembed-tests, openrouter-tests, mcp-test, relational-db-migration-tests, @@ -163,6 +170,7 @@ jobs: db-examples-tests, mcp-test, gemini-tests, + fastembed-tests, openrouter-tests, ollama-tests, relational-db-migration-tests, @@ -185,6 +193,7 @@ jobs: "${{ needs.db-examples-tests.result }}" == "success" && "${{ needs.relational-db-migration-tests.result }}" == "success" && "${{ needs.gemini-tests.result }}" == "success" && + "${{ needs.fastembed-tests.result }}" == "success" && "${{ needs.openrouter-tests.result }}" == "success" && "${{ needs.docker-compose-test.result }}" == "success" && "${{ needs.docker-ci-test.result }}" == "success" && From 70ee19616621a8ded2fec6fc2efb1fb6c1d0a849 Mon Sep 17 00:00:00 2001 From: Andrej Milicevic Date: Thu, 11 Sep 2025 18:20:11 +0200 Subject: [PATCH 2/6] Change amount of tests called in test-suites.yml --- .github/workflows/test_fastembed.yml | 2 +- .github/workflows/test_llms.yml | 25 ++++++++++++++++++++++ .github/workflows/test_suites.yml | 32 ++++++---------------------- 3 files changed, 33 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/test_llms.yml diff --git a/.github/workflows/test_fastembed.yml b/.github/workflows/test_fastembed.yml index 5d37c1118..787a270d6 100644 --- a/.github/workflows/test_fastembed.yml +++ b/.github/workflows/test_fastembed.yml @@ -20,7 +20,7 @@ jobs: env: LLM_PROVIDER: "openai" LLM_API_KEY: ${{ secrets.LLM_API_KEY }} - LLM_MODEL: "gpt-4o-mini" + LLM_MODEL: ${{ secrets.LLM_MODEL }} EMBEDDING_PROVIDER: "fastembed" EMBEDDING_MODEL: "sentence-transformers/all-MiniLM-L6-v2" EMBEDDING_DIMENSIONS: "384" diff --git a/.github/workflows/test_llms.yml b/.github/workflows/test_llms.yml new file mode 100644 index 000000000..8d493f0e7 --- /dev/null +++ b/.github/workflows/test_llms.yml @@ -0,0 +1,25 @@ +name: LLM Test Suites + +on: + workflow_call: + workflow_dispatch: + +env: + RUNTIME__LOG_LEVEL: ERROR + ENV: 'dev' + +jobs: + gemini-tests: + name: Gemini Tests + uses: ./.github/workflows/test_gemini.yml + secrets: inherit + + fastembed-tests: + name: Fastembed Tests + uses: ./.github/workflows/test_fastembed.yml + secrets: inherit + + openrouter-tests: + name: OpenRouter Tests + uses: ./.github/workflows/test_openrouter.yml + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/test_suites.yml b/.github/workflows/test_suites.yml index 06e3e10da..f051a31b3 100644 --- a/.github/workflows/test_suites.yml +++ b/.github/workflows/test_suites.yml @@ -115,22 +115,10 @@ jobs: secrets: inherit # Additional LLM tests - gemini-tests: - name: Gemini Tests - needs: [basic-tests, e2e-tests, cli-tests] - uses: ./.github/workflows/test_gemini.yml - secrets: inherit - - fastembed-tests: - name: Fastembed Tests - needs: [basic-tests, e2e-tests, cli-tests] - uses: ./.github/workflows/test_fastembed.yml - secrets: inherit - - openrouter-tests: - name: OpenRouter Tests - needs: [basic-tests, e2e-tests, cli-tests] - uses: ./.github/workflows/test_openrouter.yml + llm-tests: + name: LLM Test Suite + needs: [ basic-tests, e2e-tests, cli-tests ] + uses: ./.github/workflows/test_llms.yml secrets: inherit # Ollama tests moved to the end @@ -145,9 +133,7 @@ jobs: different-operating-systems-tests, vector-db-tests, example-tests, - gemini-tests, - fastembed-tests, - openrouter-tests, + llm-tests, mcp-test, relational-db-migration-tests, docker-compose-test, @@ -169,9 +155,7 @@ jobs: example-tests, db-examples-tests, mcp-test, - gemini-tests, - fastembed-tests, - openrouter-tests, + llm-tests, ollama-tests, relational-db-migration-tests, docker-compose-test, @@ -192,9 +176,7 @@ jobs: "${{ needs.example-tests.result }}" == "success" && "${{ needs.db-examples-tests.result }}" == "success" && "${{ needs.relational-db-migration-tests.result }}" == "success" && - "${{ needs.gemini-tests.result }}" == "success" && - "${{ needs.fastembed-tests.result }}" == "success" && - "${{ needs.openrouter-tests.result }}" == "success" && + "${{ needs.llm-tests.result }}" == "success" && "${{ needs.docker-compose-test.result }}" == "success" && "${{ needs.docker-ci-test.result }}" == "success" && "${{ needs.ollama-tests.result }}" == "success" ]]; then From 300d358d4fb867457d03cbe9ef3839a8b49ddf61 Mon Sep 17 00:00:00 2001 From: Andrej Milicevic Date: Thu, 11 Sep 2025 18:23:46 +0200 Subject: [PATCH 3/6] test name change --- .github/workflows/test_fastembed.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_fastembed.yml b/.github/workflows/test_fastembed.yml index 787a270d6..6c85d11d4 100644 --- a/.github/workflows/test_fastembed.yml +++ b/.github/workflows/test_fastembed.yml @@ -4,7 +4,7 @@ on: workflow_call: jobs: - test-gemini: + test-fastembed: name: Run Fastembed Test runs-on: ubuntu-22.04 steps: From b0c56207027e2f0bd17635eebde855568ab6c791 Mon Sep 17 00:00:00 2001 From: Andrej Milicevic Date: Fri, 12 Sep 2025 09:47:27 +0200 Subject: [PATCH 4/6] Change how llm tests are called in workflow. Delete unnecessary workflows. --- .../disable_independent_workflows.sh | 3 +- .github/workflows/test_fastembed.yml | 28 ------- .github/workflows/test_gemini.yml | 29 ------- .github/workflows/test_llms.yml | 82 ++++++++++++++++--- .github/workflows/test_openrouter.yml | 30 ------- 5 files changed, 70 insertions(+), 102 deletions(-) delete mode 100644 .github/workflows/test_fastembed.yml delete mode 100644 .github/workflows/test_gemini.yml delete mode 100644 .github/workflows/test_openrouter.yml diff --git a/.github/workflows/disable_independent_workflows.sh b/.github/workflows/disable_independent_workflows.sh index 9ec4a711e..ff57da80d 100755 --- a/.github/workflows/disable_independent_workflows.sh +++ b/.github/workflows/disable_independent_workflows.sh @@ -10,8 +10,7 @@ WORKFLOWS=( "test_kuzu.yml" "test_multimetric_qa_eval_run.yaml" "test_graphrag_vs_rag_notebook.yml" - "test_gemini.yml" - "test_fastembed.yml" + "test_llms.yml" "test_multimedia_example.yaml" "test_deduplication.yml" "test_eval_framework.yml" diff --git a/.github/workflows/test_fastembed.yml b/.github/workflows/test_fastembed.yml deleted file mode 100644 index 6c85d11d4..000000000 --- a/.github/workflows/test_fastembed.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: test | fastembed - -on: - workflow_call: - -jobs: - test-fastembed: - name: Run Fastembed Test - runs-on: ubuntu-22.04 - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Cognee Setup - uses: ./.github/actions/cognee_setup - with: - python-version: '3.11.x' - - - name: Run Fastembed Simple Example - env: - LLM_PROVIDER: "openai" - LLM_API_KEY: ${{ secrets.LLM_API_KEY }} - LLM_MODEL: ${{ secrets.LLM_MODEL }} - EMBEDDING_PROVIDER: "fastembed" - EMBEDDING_MODEL: "sentence-transformers/all-MiniLM-L6-v2" - EMBEDDING_DIMENSIONS: "384" - EMBEDDING_MAX_TOKENS: "256" - run: uv run python ./examples/python/simple_example.py diff --git a/.github/workflows/test_gemini.yml b/.github/workflows/test_gemini.yml deleted file mode 100644 index 544e15a5e..000000000 --- a/.github/workflows/test_gemini.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: test | gemini - -on: - workflow_call: - -jobs: - test-gemini: - name: Run Gemini Test - runs-on: ubuntu-22.04 - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Cognee Setup - uses: ./.github/actions/cognee_setup - with: - python-version: '3.11.x' - - - name: Run Gemini Simple Example - env: - LLM_PROVIDER: "gemini" - LLM_API_KEY: ${{ secrets.GEMINI_API_KEY }} - LLM_MODEL: "gemini/gemini-1.5-flash" - EMBEDDING_PROVIDER: "gemini" - EMBEDDING_API_KEY: ${{ secrets.GEMINI_API_KEY }} - EMBEDDING_MODEL: "gemini/text-embedding-004" - EMBEDDING_DIMENSIONS: "768" - EMBEDDING_MAX_TOKENS: "8076" - run: uv run python ./examples/python/simple_example.py diff --git a/.github/workflows/test_llms.yml b/.github/workflows/test_llms.yml index 8d493f0e7..b70b2e5ed 100644 --- a/.github/workflows/test_llms.yml +++ b/.github/workflows/test_llms.yml @@ -2,24 +2,80 @@ name: LLM Test Suites on: workflow_call: - workflow_dispatch: env: RUNTIME__LOG_LEVEL: ERROR ENV: 'dev' jobs: - gemini-tests: - name: Gemini Tests - uses: ./.github/workflows/test_gemini.yml - secrets: inherit + test-gemini: + name: Run Gemini Test + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v4 - fastembed-tests: - name: Fastembed Tests - uses: ./.github/workflows/test_fastembed.yml - secrets: inherit + - name: Cognee Setup + uses: ./.github/actions/cognee_setup + with: + python-version: '3.11.x' - openrouter-tests: - name: OpenRouter Tests - uses: ./.github/workflows/test_openrouter.yml - secrets: inherit \ No newline at end of file + - name: Run Gemini Simple Example + env: + LLM_PROVIDER: "gemini" + LLM_API_KEY: ${{ secrets.GEMINI_API_KEY }} + LLM_MODEL: "gemini/gemini-1.5-flash" + EMBEDDING_PROVIDER: "gemini" + EMBEDDING_API_KEY: ${{ secrets.GEMINI_API_KEY }} + EMBEDDING_MODEL: "gemini/text-embedding-004" + EMBEDDING_DIMENSIONS: "768" + EMBEDDING_MAX_TOKENS: "8076" + run: uv run python ./examples/python/simple_example.py + + test-fastembed: + name: Run Fastembed Test + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Cognee Setup + uses: ./.github/actions/cognee_setup + with: + python-version: '3.11.x' + + - name: Run Fastembed Simple Example + env: + LLM_PROVIDER: "openai" + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + LLM_MODEL: ${{ secrets.LLM_MODEL }} + EMBEDDING_PROVIDER: "fastembed" + EMBEDDING_MODEL: "sentence-transformers/all-MiniLM-L6-v2" + EMBEDDING_DIMENSIONS: "384" + EMBEDDING_MAX_TOKENS: "256" + run: uv run python ./examples/python/simple_example.py + + test-openrouter: + name: Run OpenRouter Test + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Cognee Setup + uses: ./.github/actions/cognee_setup + with: + python-version: '3.11.x' + + - name: Run OpenRouter Simple Example + env: + LLM_PROVIDER: "custom" + LLM_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + LLM_MODEL: "openrouter/x-ai/grok-code-fast-1" + LLM_ENDPOINT: "https://openrouter.ai/api/v1" + EMBEDDING_PROVIDER: "openai" + EMBEDDING_API_KEY: ${{ secrets.OPENAI_API_KEY }} + EMBEDDING_MODEL: "openai/text-embedding-3-large" + EMBEDDING_DIMENSIONS: "3072" + EMBEDDING_MAX_TOKENS: "8191" + run: uv run python ./examples/python/simple_example.py \ No newline at end of file diff --git a/.github/workflows/test_openrouter.yml b/.github/workflows/test_openrouter.yml deleted file mode 100644 index 9c2dcdebe..000000000 --- a/.github/workflows/test_openrouter.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: test | openrouter - -on: - workflow_call: - -jobs: - test-openrouter: - name: Run OpenRouter Test - runs-on: ubuntu-22.04 - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Cognee Setup - uses: ./.github/actions/cognee_setup - with: - python-version: '3.11.x' - - - name: Run OpenRouter Simple Example - env: - LLM_PROVIDER: "custom" - LLM_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - LLM_MODEL: "openrouter/x-ai/grok-code-fast-1" - LLM_ENDPOINT: "https://openrouter.ai/api/v1" - EMBEDDING_PROVIDER: "openai" - EMBEDDING_API_KEY: ${{ secrets.OPENAI_API_KEY }} - EMBEDDING_MODEL: "openai/text-embedding-3-large" - EMBEDDING_DIMENSIONS: "3072" - EMBEDDING_MAX_TOKENS: "8191" - run: uv run python ./examples/python/simple_example.py From 82afdef484bbfcc38cea50a326ab72bd3052db06 Mon Sep 17 00:00:00 2001 From: Igor Ilic <30923996+dexters1@users.noreply.github.com> Date: Fri, 12 Sep 2025 07:27:31 -0400 Subject: [PATCH 5/6] Potential fix for code scanning alert no. 227: Workflow does not contain permissions Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/test_llms.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_llms.yml b/.github/workflows/test_llms.yml index b70b2e5ed..77ee4e2e7 100644 --- a/.github/workflows/test_llms.yml +++ b/.github/workflows/test_llms.yml @@ -1,5 +1,8 @@ name: LLM Test Suites +permissions: + contents: read + on: workflow_call: From a20fe8ad3471e1bdd8bdccebb26afcb8005888b9 Mon Sep 17 00:00:00 2001 From: Andrej Milicevic Date: Fri, 12 Sep 2025 15:15:06 +0200 Subject: [PATCH 6/6] fix Azure error in test --- .github/workflows/test_llms.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_llms.yml b/.github/workflows/test_llms.yml index 77ee4e2e7..5a0f947c9 100644 --- a/.github/workflows/test_llms.yml +++ b/.github/workflows/test_llms.yml @@ -52,6 +52,8 @@ jobs: LLM_PROVIDER: "openai" LLM_API_KEY: ${{ secrets.LLM_API_KEY }} LLM_MODEL: ${{ secrets.LLM_MODEL }} + LLM_ENDPOINT: ${{ secrets.LLM_ENDPOINT }} + LLM_API_VERSION: ${{ secrets.LLM_API_VERSION }} EMBEDDING_PROVIDER: "fastembed" EMBEDDING_MODEL: "sentence-transformers/all-MiniLM-L6-v2" EMBEDDING_DIMENSIONS: "384"