Merge pull request #111 from topoteretes/fix/simple-docker

Fix/simple docker

.dockerignore

@@ -0,0 +1,16 @@
+bin
+dist
+docs
+evals
+tests
+tools
+assets
+notebooks
+cognee-frontend
+
+.dlt
+.venv
+.github
+
+.idea
+.vscode

.github/workflows/cd.yaml

@@ -1,4 +1,4 @@
-name: publish dev |  Docker image
+name: publish dev | Docker image
 
 on:
   push:

.github/workflows/cd_prd.yaml

@@ -1,4 +1,4 @@
-name: publish prd |  Docker image
+name: publish prd | Docker image
 
 on:
   push:

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name: build test |  Docker image
+name: build test | Docker image
 
 on: pull_request
 

.github/workflows/docker_compose.yml

@@ -0,0 +1,44 @@
+name: test | docker compose
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  docker-compose-test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Build Docker images
+      run: |
+        docker-compose -f docker-compose.yml build
+
+    - name: Run Docker Compose
+      run: |
+        docker-compose -f docker-compose.yml up -d
+
+    - name: Wait for services to be ready
+      run: |
+        # Add any necessary health checks or wait commands
+        sleep 30
+
+    # - name: Run tests
+    #   run: |
+    #     docker-compose -f docker-compose.yml run --rm <test-service> <test-command>
+    #   # Replace <test-service> with the name of the service running the tests
+    #   # Replace <test-command> with the actual command to run your tests
+
+    - name: Shut down Docker Compose
+      if: always()
+      run: |
+        docker-compose -f docker-compose.yml down

.github/workflows/test_neo4j.yml

@@ -1,4 +1,4 @@
-name: test neo4j
+name: test | neo4j
 
 on:
   pull_request:
@@ -18,7 +18,7 @@ jobs:
     name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
-  run_common:
+  run_neo4j_integration_test:
     name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'

Dockerfile

@@ -1,51 +1,29 @@
-FROM python:3.11
+FROM python:3.11-slim
 
 # Set build argument
 ARG DEBUG
-ARG API_ENABLED
 
 # Set environment variable based on the build argument
 ENV DEBUG=${DEBUG}
-ENV API_ENABLED=${API_ENABLED}
 ENV PIP_NO_CACHE_DIR=true
 ENV PATH="${PATH}:/root/.poetry/bin"
 
-RUN pip install poetry
-
 WORKDIR /app
 COPY pyproject.toml poetry.lock /app/
 
+RUN pip install poetry
+
+# Create virtualenv
+RUN poetry config virtualenvs.create false
+
 # Install the dependencies
-RUN poetry config virtualenvs.create false && \
-    poetry install --no-root --no-dev
-
-RUN apt-get update -q && \
-    apt-get install -y -q \
-        gcc \
-        python3-dev \
-        curl \
-        zip \
-        jq \
-#        libgl1-mesa-glx \
-        netcat-traditional && \
-    pip install poetry && \
-    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
-    unzip -qq awscliv2.zip && \
-    ./aws/install && \
-    apt-get clean && \
-    rm -rf \
-        awscliv2.zip \
-        /var/lib/apt/lists/* \
-        /tmp/* \
-        /var/tmp/*
-
-WORKDIR /app
+RUN poetry install --no-root --no-dev
+        
 # Set the PYTHONPATH environment variable to include the /app directory
 ENV PYTHONPATH=/app
 
 COPY cognee/ /app/cognee
 
-
 COPY entrypoint.sh /app/entrypoint.sh
 RUN chmod +x /app/entrypoint.sh
 

Dockerfile-old

@@ -0,0 +1,49 @@
+FROM python:3.11
+
+# Set build argument
+ARG DEBUG
+
+# Set environment variable based on the build argument
+ENV DEBUG=${DEBUG}
+ENV PIP_NO_CACHE_DIR=true
+ENV PATH="${PATH}:/root/.poetry/bin"
+
+RUN pip install poetry
+
+WORKDIR /app
+COPY pyproject.toml poetry.lock /app/
+
+# Install the dependencies
+RUN poetry config virtualenvs.create false && \
+    poetry install --no-root --no-dev
+
+RUN apt-get update -q && \
+    apt-get install -y -q \
+        gcc \
+        python3-dev \
+        curl \
+        zip \
+        jq \
+        netcat-traditional && \
+    pip install poetry && \
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip -qq awscliv2.zip && \
+    ./aws/install && \
+    apt-get clean && \
+    rm -rf \
+        awscliv2.zip \
+        /var/lib/apt/lists/* \
+        /tmp/* \
+        /var/tmp/*
+
+WORKDIR /app
+# Set the PYTHONPATH environment variable to include the /app directory
+ENV PYTHONPATH=/app
+
+COPY cognee/ /app/cognee
+
+
+COPY entrypoint.sh /app/entrypoint.sh
+RUN chmod +x /app/entrypoint.sh
+
+ENTRYPOINT ["/app/entrypoint.sh"]

cognee-frontend/.dockerignore

@@ -0,0 +1,2 @@
+.next
+node_modules

cognee-frontend/Dockerfile

@@ -1,20 +1,20 @@
 # Use an official Node.js runtime as a parent image
-FROM node:18.17.0
+FROM node:18-alpine
 
 # Set the working directory to /app
 WORKDIR /app
 
 # Copy package.json and package-lock.json to the working directory
-COPY package*.json ./
+COPY package.json package-lock.json ./
 
 # Install any needed packages specified in package.json
-RUN npm install
+RUN npm ci
 
 # Copy the rest of the application code to the working directory
-COPY . .
-
-# Make port 3000 available to the world outside this container
-EXPOSE 3000
+COPY src ./src
+COPY public ./public
+COPY next.config.mjs .
+COPY tsconfig.json .
 
 # Build the app and run it
-CMD ["sh", "-c", "npm run dev"]
+CMD npm run dev

cognee/tests/test_qdrant.py

@@ -73,4 +73,4 @@ async def main():
 
 if __name__ == "__main__":
     import asyncio
-    asyncio.run(main())
+    asyncio.run(main())

docker-compose-vanilla.yml

@@ -0,0 +1,41 @@
+services:
+  cognee:
+    container_name: cognee
+    networks:
+      - cognee-network
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - .:/app
+      - /app/cognee-frontend/ # Ignore frontend code
+    environment:
+      - HOST=0.0.0.0
+      - ENVIRONMENT=local
+      - PYTHONPATH=.
+    ports:
+      - 8000:8000
+      # - 5678:5678 # Debugging
+    deploy:
+      resources:
+        limits:
+          cpus: 4.0
+          memory: 8GB
+
+  frontend:
+    container_name: frontend
+    build:
+      context: ./cognee-frontend
+      dockerfile: Dockerfile
+    volumes:
+      - ./cognee-frontend/src:/app/src
+      - ./cognee-frontend/public:/app/public
+    ports:
+      - 3000:3000
+      # - 9229:9229 # Debugging
+    networks:
+      - cognee-network
+
+networks:
+  cognee-network:
+    name: cognee-network

docker-compose.yml

@@ -1,47 +1,25 @@
-version: "3.9"
-
 services:
-  neo4j:
-    image: neo4j:latest
-    container_name: neo4j
-    ports:
-      - "7474:7474"
-      - "7687:7687"
-    environment:
-      - NEO4J_AUTH=neo4j/pleaseletmein
-      - NEO4J_PLUGINS=["apoc"]
-    networks:
-      - cognee_backend
-
   cognee:
     container_name: cognee
     networks:
-      - cognee_backend
+      - cognee-network
     build:
-      context: ./
+      context: .
+      dockerfile: Dockerfile
     volumes:
-      - "./:/app"
-      - ./.data:/app/.data
+      - .:/app
+      - /app/cognee-frontend/ # Ignore frontend code
     environment:
       - HOST=0.0.0.0
       - ENVIRONMENT=local
       - PYTHONPATH=.
-    profiles: ["exclude-from-up"]
     ports:
       - 8000:8000
-      - 443:443
-      - 80:80
-      - 50051:50051
-      - 5678:5678
-#      - 5432:5432
-    depends_on:
-      - postgres
-      - neo4j
-      - frontend
+      # - 5678:5678 # Debugging
     deploy:
       resources:
         limits:
-          cpus: "4.0"
+          cpus: 4.0
           memory: 8GB
 
   frontend:
@@ -50,36 +28,40 @@ services:
       context: ./cognee-frontend
       dockerfile: Dockerfile
     volumes:
-      - "./cognee-frontend:/app"
+      - ./cognee-frontend/src:/app/src
+      - ./cognee-frontend/public:/app/public
     ports:
-      - "3000:3000"
+      - 3000:3000
+      # - 9229:9229 # Debugging
     networks:
-      - cognee_backend
+      - cognee-network
 
-  postgres:
-     image: postgres
-     container_name: postgres
-     environment:
-       - POSTGRES_HOST_AUTH_METHOD=trust
-       - POSTGRES_USER=bla
-       - POSTGRES_PASSWORD=bla
-       - POSTGRES_DB=bubu
-
-     networks:
-       - cognee_backend
-     ports:
-       - "5432:5432"
+  neo4j:
+    image: neo4j:latest
+    container_name: neo4j
+    profiles:
+      - neo4j
+    ports:
+      - 7474:7474
+      - 7687:7687
+    environment:
+      - NEO4J_AUTH=neo4j/pleaseletmein
+      - NEO4J_PLUGINS=["apoc"]
+    networks:
+      - cognee-network
 
   falkordb:
     image: falkordb/falkordb:edge
     container_name: falkordb
+    profiles:
+      - falkordb
     ports:
-      - "6379:6379"
-      - "3001:3000"
+      - 6379:6379
+      - 3001:3000
     networks:
-      - cognee_backend
+      - cognee-network
 
 networks:
-  cognee_backend:
-    name: cognee_backend
+  cognee-network:
+    name: cognee-network
 

entrypoint-old.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# export ENVIRONMENT
+
+echo "Debug mode: $DEBUG"
+echo "Environment: $ENVIRONMENT"
+
+if [ "$ENVIRONMENT" != "local" ]; then
+    echo "Running fetch_secret.py"
+
+    PYTHONPATH=. python cognee/fetch_secret.py
+
+    if [ $? -ne 0 ]; then
+        echo "Error: fetch_secret.py failed"
+        exit 1
+    fi
+else
+    echo '"local" environment is active, skipping fetch_secret.py'
+fi
+
+echo "Creating database..."
+#
+#PYTHONPATH=. python cognee/setup_database.py
+#if [ $? -ne 0 ]; then
+#    echo "Error: setup_database.py failed"
+#    exit 1
+#fi
+
+echo "Starting Gunicorn"
+
+if [ "$DEBUG" = true ]; then
+  echo "Waiting for the debugger to attach..."
+  python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m gunicorn -w 3 -k uvicorn.workers.UvicornWorker -t 30000 --bind=0.0.0.0:8000 --bind=0.0.0.0:443 --log-level debug cognee.api.client:app
+else
+  gunicorn -w 3 -k uvicorn.workers.UvicornWorker -t 30000 --bind=0.0.0.0:8000 --bind=0.0.0.0:443 --log-level debug cognee.api.client:app
+fi

entrypoint.sh

@@ -1,35 +1,18 @@
 #!/bin/bash
-# export ENVIRONMENT
 
 echo "Debug mode: $DEBUG"
 echo "Environment: $ENVIRONMENT"
 
-if [ "$ENVIRONMENT" != "local" ]; then
-    echo "Running fetch_secret.py"
-
-    PYTHONPATH=. python cognee/fetch_secret.py
-
-    if [ $? -ne 0 ]; then
-        echo "Error: fetch_secret.py failed"
-        exit 1
-    fi
-else
-    echo '"local" environment is active, skipping fetch_secret.py'
-fi
-
-echo "Creating database..."
-#
-#PYTHONPATH=. python cognee/setup_database.py
-#if [ $? -ne 0 ]; then
-#    echo "Error: setup_database.py failed"
-#    exit 1
-#fi
-
 echo "Starting Gunicorn"
 
-if [ "$DEBUG" = true ]; then
-  echo "Waiting for the debugger to attach..."
-  python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m gunicorn -w 3 -k uvicorn.workers.UvicornWorker -t 30000 --bind=0.0.0.0:8000 --bind=0.0.0.0:443 --log-level debug cognee.api.client:app
+if [ "$ENVIRONMENT" = "local" ]; then
+  if [ "$DEBUG" = true ]; then
+    echo "Waiting for the debugger to attach..."
+
+    python -m debugpy --wait-for-client --listen 0.0.0.0:5678 -m gunicorn -w 3 -k uvicorn.workers.UvicornWorker -t 30000 --bind=0.0.0.0:8000 --log-level debug --reload cognee.api.client:app
+  else
+    gunicorn -w 3 -k uvicorn.workers.UvicornWorker -t 30000 --bind=0.0.0.0:8000 --log-level debug --reload cognee.api.client:app
+  fi
 else
-  gunicorn -w 3 -k uvicorn.workers.UvicornWorker -t 30000 --bind=0.0.0.0:8000 --bind=0.0.0.0:443 --log-level debug cognee.api.client:app
+  gunicorn -w 3 -k uvicorn.workers.UvicornWorker -t 30000 --bind=0.0.0.0:8000 --log-level error cognee.api.client:app
 fi

poetry.lock

@@ -8314,14 +8314,13 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools",
 cli = []
 duckdb = ["duckdb"]
 filesystem = []
-motherduck = ["duckdb", "pyarrow"]
+motherduck = ["duckdb"]
 neo4j = ["neo4j"]
 notebook = ["overrides"]
-parquet = ["pyarrow"]
 qdrant = ["qdrant-client"]
 weaviate = ["weaviate-client"]
 
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9.0,<3.12"
-content-hash = "0d47b955224bb4f7d52815e63d15b2230dcff86f91a4f79b5533c299f965018b"
+content-hash = "3e894bd32ee1d47a344ad4325731c064271b687b1f6af3c820436dc27f9d06a1"

pyproject.toml

@@ -30,7 +30,7 @@ sqlalchemy = "^2.0.21"
 instructor = "1.2.1"
 networkx = "^3.2.1"
 debugpy = "^1.8.0"
-pyarrow = "^15.0.0"
+pyarrow = "15.0.0"
 pylint = "^3.0.3"
 aiosqlite = "^0.20.0"
 pandas = "2.0.3"
@@ -75,12 +75,10 @@ anthropic = "^0.26.1"
 langchain-text-splitters = "^0.2.1"
 
 
-
 [tool.poetry.extras]
-parquet = ["pyarrow"]
 duckdb = ["duckdb"]
 filesystem = ["s3fs", "botocore"]
-motherduck = ["duckdb", "pyarrow"]
+motherduck = ["duckdb"]
 cli = ["pipdeptree", "cron-descriptor"]
 weaviate = ["weaviate-client"]
 qdrant = ["qdrant-client"]