feat: enable feeeback and activity log

cognee-frontend/src/app/(graph)/CogneeAddWidget.tsx

@@ -1,5 +1,6 @@
 "use client";
 
+import { v4 as uuid4 } from "uuid";
 import { ChangeEvent, useEffect } from "react";
 import { CTAButton, StatusIndicator } from "@/ui/elements";
 
@@ -30,15 +31,16 @@ export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
       .then((datasets) => {
         const dataset = datasets?.[0];
 
-        if (dataset) {
+        // For CrewAI we don't have a dataset.
-          getDatasetGraph(dataset)
+        // if (dataset) {
-            .then((graph) => onData({
+        getDatasetGraph(dataset || { id: uuid4() })
-              nodes: graph.nodes,
+          .then((graph) => onData({
-              links: graph.edges,
+            nodes: graph.nodes,
-            }));
+            links: graph.edges,
-        }
+          }));
+        // }
       });
-  }, [refreshDatasets]);
+  }, [onData, refreshDatasets]);
 
   const handleAddFiles = (dataset: { id?: string, name?: string }, event: ChangeEvent<HTMLInputElement>) => {
     event.stopPropagation();
@@ -51,8 +53,6 @@ export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
 
     return addData(dataset, files)
       .then(() => {
-        console.log("Data added successfully.");
-
         const onUpdate = (data: any) => {
           onData({
             nodes: data.payload.nodes,
@@ -60,11 +60,12 @@ export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
           });
         };
 
-        return cognifyDataset(dataset, onUpdate)
+        return cognifyDataset(dataset, onUpdate);
-          .then((data) => console.log(data));
       });
   };
 
+  return null;
+
   return (
     <div className="flex flex-col gap-4 mb-4">
       {datasets.length ? datasets.map((dataset) => (

cognee-frontend/src/app/(graph)/CrewAITrigger.tsx

@@ -1,4 +1,7 @@
+import { useState } from "react";
 import { fetch } from "@/utils";
+import { v4 as uuid4 } from "uuid";
+import { LoadingIndicator } from "@/ui/App";
 import { CTAButton, Input } from "@/ui/elements";
 
 interface CrewAIFormPayload extends HTMLFormElement {
@@ -6,21 +9,98 @@ interface CrewAIFormPayload extends HTMLFormElement {
   username2: HTMLInputElement;
 }
 
-export default function CrewAITrigger() {
+interface CrewAITriggerProps {
+  onData: (data: any) => void;
+  onActivity: (activities: any) => void;
+}
+
+export default function CrewAITrigger({ onData, onActivity }: CrewAITriggerProps) {
+  const [isCrewAIRunning, setIsCrewAIRunning] = useState(false);
+  
   const handleRunCrewAI = (event: React.FormEvent<CrewAIFormPayload>) => {
-    fetch("/v1/crew-ai/run", {
+    event.preventDefault();
+    const formElements = event.currentTarget;
+
+    const crewAIConfig = {
+      username1: formElements.username1.value,
+      username2: formElements.username2.value,
+    };
+
+    const websocket = new WebSocket("ws://localhost:8000/api/v1/crewai/subscribe");
+
+    websocket.onopen = () => {
+      websocket.send(JSON.stringify({
+        "Authorization": `Bearer ${localStorage.getItem("access_token")}`,
+      }));
+    };
+
+    let isCrewAIDone = false;
+    onActivity([{ id: uuid4(), timestamp: Date.now(), activity: "Running CrewAI" }]);
+
+    websocket.onmessage = (event) => {
+      const data = JSON.parse(event.data);
+
+      if (data.status === "PipelineRunActivity") {
+        onActivity(data.payload);
+        return;
+      }
+
+      onData({
+        nodes: data.payload.nodes,
+        links: data.payload.edges,
+      });
+
+      const nodes_type_map: { [key: string]: number } = {};
+
+      for (let i = 0; i < data.payload.nodes.length; i++) {
+        const node = data.payload.nodes[i];
+        if (!nodes_type_map[node.type]) {
+          nodes_type_map[node.type] = 0;
+        }
+        nodes_type_map[node.type] += 1;
+      }
+
+      const activityMessage = Object.entries(nodes_type_map).reduce((message, [type, count]) => {
+        return `${message}\n | ${type}: ${count}`;
+      }, "Graph updated:");
+
+      onActivity([{
+        id: uuid4(),
+        timestamp: Date.now(),
+        activity: activityMessage,
+      }]);
+
+      if (data.status === "PipelineRunCompleted") {
+        isCrewAIDone = true;
+        websocket.close();
+      }
+    };
+
+    setIsCrewAIRunning(true);
+
+    return fetch("/v1/crewai/run", {
       method: "POST",
-      body: new FormData(event.currentTarget),
+      body: JSON.stringify(crewAIConfig),
+      headers: {
+        "Content-Type": "application/json",
+      },
     })
       .then(response => response.json())
-      .then((data) => console.log(data));
+      .finally(() => {
+        websocket.close();
+        setIsCrewAIRunning(false);
+        onActivity([{ id: uuid4(), timestamp: Date.now(), activity: "CrewAI run done" }]);
+      });
   };
 
   return (
     <form className="w-full flex flex-row gap-2 items-center" onSubmit={handleRunCrewAI}>
-      <Input type="text" placeholder="Github Username" required />
+      <Input name="username1" type="text" placeholder="Github Username" required defaultValue="hajdul88" />
-      <Input type="text" placeholder="Github Username" required />
+      <Input name="username2" type="text" placeholder="Github Username" required defaultValue="lxobr" />
-      <CTAButton type="submit" className="whitespace-nowrap">Run CrewAI</CTAButton>
+      <CTAButton type="submit" className="whitespace-nowrap">
+        Run CrewAI
+        {isCrewAIRunning && <LoadingIndicator />}
+      </CTAButton>
     </form>
   );
 }

cognee-frontend/src/app/(graph)/GraphControls.tsx

@@ -19,31 +19,38 @@ interface GraphControlsProps {
 export interface GraphControlsAPI {
   setSelectedNode: (node: NodeObject | null) => void;
   getSelectedNode: () => NodeObject | null;
+  updateActivity: (activities: ActivityLog[]) => void;
 }
 
 type ActivityLog = {
   id: string;
   timestamp: number;
   activity: string;
-}[];
+};
 
-type NodeProperties = {
+type NodeProperty = {
   id: string;
   name: string;
   value: string;
-}[];
+};
+
+const formatter = new Intl.DateTimeFormat("en-GB", { dateStyle: "short", timeStyle: "medium" });
 
 export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, onFitIntoView, ref }: GraphControlsProps) {
   const [selectedNode, setSelectedNode] = useState<NodeObject | null>(null);
-  const [activityLog, setActivityLog] = useState<ActivityLog>([]);
+  const [activityLog, setActivityLog] = useState<ActivityLog[]>([]);
-  const [nodeProperties, setNodeProperties] = useState<NodeProperties>([]);
+  const [nodeProperties, setNodeProperties] = useState<NodeProperty[]>([]);
-  const [newProperty, setNewProperty] = useState<NodeProperties[0]>({
+  const [newProperty, setNewProperty] = useState<NodeProperty>({
     id: uuid4(),
     name: "",
     value: "",
   });
 
-  const handlePropertyChange = (property: NodeProperties[0], property_key: string, event: ChangeEvent<HTMLInputElement>) => {
+  const updateActivity = (newActivities: ActivityLog[]) => {
+    setActivityLog((activities) => [...activities, ...newActivities]);
+  };
+
+  const handlePropertyChange = (property: NodeProperty, property_key: string, event: ChangeEvent<HTMLInputElement>) => {
     const value = event.target.value;
 
     setNodeProperties(nodeProperties.map((nodeProperty) => (nodeProperty.id === property.id ? {...nodeProperty, [property_key]: value } : nodeProperty)));
@@ -58,11 +65,11 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
     }
   };
 
-  const handlePropertyDelete = (property: NodeProperties[0]) => {
+  const handlePropertyDelete = (property: NodeProperty) => {
     setNodeProperties(nodeProperties.filter((nodeProperty) => nodeProperty.id !== property.id));
   };
 
-  const handleNewPropertyChange = (property: NodeProperties[0], property_key: string, event: ChangeEvent<HTMLInputElement>) => {
+  const handleNewPropertyChange = (property: NodeProperty, property_key: string, event: ChangeEvent<HTMLInputElement>) => {
     const value = event.target.value;
 
     setNewProperty({...property, [property_key]: value });
@@ -71,6 +78,7 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
   useImperativeHandle(ref, () => ({
     setSelectedNode,
     getSelectedNode: () => selectedNode,
+    updateActivity,
   }));
 
   const [selectedTab, setSelectedTab] = useState("nodeDetails");
@@ -81,14 +89,14 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
 
   return (
     <>
-      <div className="flex">
+      <div className="flex w-full">
-        <button onClick={() => setSelectedTab("nodeDetails")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30", { "border-b-indigo-600 text-white": selectedTab === "nodeDetails" })}>
+        <button onClick={() => setSelectedTab("nodeDetails")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30 flex-1/3", { "border-b-indigo-600 text-white": selectedTab === "nodeDetails" })}>
           <span className="whitespace-nowrap">Node Details</span>
         </button>
-        <button onClick={() => setSelectedTab("activityLog")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30", { "border-b-indigo-600 text-white": selectedTab === "activityLog" })}>
+        <button onClick={() => setSelectedTab("activityLog")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30 flex-1/3", { "border-b-indigo-600 text-white": selectedTab === "activityLog" })}>
           <span className="whitespace-nowrap">Activity Log</span>
         </button>
-        <button onClick={() => setSelectedTab("feedback")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30", { "border-b-indigo-600 text-white": selectedTab === "feedback" })}>
+        <button onClick={() => setSelectedTab("feedback")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30 flex-1/3", { "border-b-indigo-600 text-white": selectedTab === "feedback" })}>
           <span className="whitespace-nowrap">Feedback</span>
         </button>
       </div>
@@ -97,9 +105,9 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
         {selectedTab === "nodeDetails" && (
           <>
             <div className="w-full flex flex-row gap-2 items-center mb-4">
-              <label className="text-gray-300 whitespace-nowrap">Graph Shape:</label>
+              <label className="text-gray-300 whitespace-nowrap flex-1/5">Graph Shape:</label>
-              <Select onChange={handleGraphShapeControl}>
+              <Select defaultValue="none" onChange={handleGraphShapeControl} className="flex-2/5">
-                <option selected value="none">None</option>
+                <option value="none">None</option>
                 <option value="td">Top-down</option>
                 <option value="bu">Bottom-up</option>
                 <option value="lr">Left-right</option>
@@ -107,9 +115,9 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
                 <option value="radialin">Radial-in</option>
                 <option value="radialout">Radial-out</option>
               </Select>
+              <NeutralButton onClick={onFitIntoView} className="flex-2/5 whitespace-nowrap">Fit Graph into View</NeutralButton>
             </div>
 
-            <NeutralButton onClick={onFitIntoView} className="mb-4">Fit Graph into View</NeutralButton>
 
             {isAddNodeFormOpen ? (
               <form className="flex flex-col gap-4" onSubmit={() => {}}>
@@ -138,21 +146,25 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
             ) : (
               selectedNode ? (
                 <div className="flex flex-col gap-4">
-                  <div className="flex flex-col gap-2">
+                  <div className="flex flex-col gap-2 overflow-y-auto max-h-96">
-                    <div className="flex gap-2 items-center">
+                    <div className="flex gap-2 items-top">
                       <span className="text-gray-300">ID:</span>
                       <span className="text-white">{selectedNode.id}</span>
                     </div>
+                    <div className="flex gap-2 items-top">
+                      <span className="text-gray-300">Label:</span>
+                      <span className="text-white">{selectedNode.label}</span>
+                    </div>
 
                     {Object.entries(selectedNode.properties).map(([key, value]) => (
-                      <div key={key} className="flex gap-2 items-center">
+                      <div key={key} className="flex gap-2 items-top">
-                        <span className="text-gray-300">{key}:</span>
+                        <span className="text-gray-300">{key.charAt(0).toUpperCase() + key.slice(1)}:</span>
-                        <span className="text-white">{typeof value === "object" ? JSON.stringify(value) : value as string}</span>
+                        <span className="text-white truncate">{typeof value === "object" ? JSON.stringify(value) : value as string}</span>
                       </div>
                     ))}
                   </div>
 
-                  <CTAButton type="button" onClick={() => {}}>Edit Node</CTAButton>
+                  {/* <CTAButton type="button" onClick={() => {}}>Edit Node</CTAButton> */}
                 </div>
               ) : (
                 <span className="text-white">No node selected.</span>
@@ -164,9 +176,9 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
         {selectedTab === "activityLog" && (
           <div className="flex flex-col gap-2">
             {activityLog.map((activity) => (
-              <div key={activity.id} className="flex gap-2 items-center">
+              <div key={activity.id} className="flex gap-2 items-top">
-                <span className="text-gray-300">{activity.timestamp}</span>
+                <span className="text-gray-300 whitespace-nowrap">{formatter.format(activity.timestamp)}: </span>
-                <span className="text-white">{activity.activity}</span>
+                <span className="text-white  whitespace-normal">{activity.activity}</span>
               </div>
             ))}
             {!activityLog.length && <span className="text-white">No activity logged.</span>}

cognee-frontend/src/app/(graph)/GraphView.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { forceCollide, forceManyBody } from "d3-force-3d";
-import { useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import ForceGraph, { ForceGraphMethods, LinkObject, NodeObject } from "react-force-graph-2d";
 
 import { TextLogo } from "@/ui/App";
@@ -35,24 +35,25 @@ export default function GraphView() {
 
   const [data, updateData] = useState<GraphData | null>(null);
 
-  const onDataChange = (newData: NodesAndEdges) => {
+  const onDataChange = useCallback((newData: NodesAndEdges) => {
-    if (data === null) {
+    if (!newData.nodes.length && !newData.links.length) {
-      updateData({
+      return;
-        nodes: newData.nodes,
-        links: newData.links,
-      });
-    } else {
-      updateData({
-        nodes: [...data.nodes, ...newData.nodes],
-        links: [...data.links, ...newData.links],
-      });
     }
-  };
+
+    updateData({
+      nodes: newData.nodes,
+      links: newData.links,
+    });
+  }, []);
 
   const graphRef = useRef<ForceGraphMethods>();
 
   const graphControls = useRef<GraphControlsAPI>(null);
 
+  const onActivityChange = (activities: any) => {
+    graphControls.current?.updateActivity(activities);
+  };
+
   const handleNodeClick = (node: NodeObject) => {
     graphControls.current?.setSelectedNode(node);
     graphRef.current?.d3ReheatSimulation();
@@ -63,29 +64,31 @@ export default function GraphView() {
   const addNodeDistanceFromSourceNode = 15;
 
   const handleBackgroundClick = (event: MouseEvent) => {
-    const graphBoundingBox = document.getElementById("graph-container")?.querySelector("canvas")?.getBoundingClientRect();
-    const x = event.clientX - graphBoundingBox!.x;
-    const y = event.clientY - graphBoundingBox!.y;
-
-    const graphClickCoords = graphRef.current!.screen2GraphCoords(x, y);
-
     const selectedNode = graphControls.current?.getSelectedNode();
 
     if (!selectedNode) {
       return;
     }
 
-    const distanceFromAddNode = Math.sqrt(
+    graphControls.current?.setSelectedNode(null);
-      Math.pow(graphClickCoords.x - (selectedNode!.x! + addNodeDistanceFromSourceNode), 2)
-      + Math.pow(graphClickCoords.y - (selectedNode!.y! + addNodeDistanceFromSourceNode), 2)
-    );
 
-    if (distanceFromAddNode <= 10) {
+    // const graphBoundingBox = document.getElementById("graph-container")?.querySelector("canvas")?.getBoundingClientRect();
-      enableAddNodeForm();
+    // const x = event.clientX - graphBoundingBox!.x;
-    } else {
+    // const y = event.clientY - graphBoundingBox!.y;
-      disableAddNodeForm();
+
-      graphControls.current?.setSelectedNode(null);
+    // const graphClickCoords = graphRef.current!.screen2GraphCoords(x, y);
-    }
+
+    // const distanceFromAddNode = Math.sqrt(
+    //   Math.pow(graphClickCoords.x - (selectedNode!.x! + addNodeDistanceFromSourceNode), 2)
+    //   + Math.pow(graphClickCoords.y - (selectedNode!.y! + addNodeDistanceFromSourceNode), 2)
+    // );
+
+    // if (distanceFromAddNode <= 10) {
+    //   enableAddNodeForm();
+    // } else {
+    //   disableAddNodeForm();
+    //   graphControls.current?.setSelectedNode(null);
+    // }
   };
 
   function renderNode(node: NodeObject, ctx: CanvasRenderingContext2D, globalScale: number) {
@@ -93,23 +96,23 @@ export default function GraphView() {
 
     ctx.save();
 
-    if (node.id === selectedNode?.id) {
+    // if (node.id === selectedNode?.id) {
-      ctx.fillStyle = "gray";
+    //   ctx.fillStyle = "gray";
 
-      ctx.beginPath();
+    //   ctx.beginPath();
-      ctx.arc(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode, 10, 0, 2 * Math.PI);
+    //   ctx.arc(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode, 10, 0, 2 * Math.PI);
-      ctx.fill();
+    //   ctx.fill();
 
-      ctx.beginPath();
+    //   ctx.beginPath();
-      ctx.moveTo(node.x! + addNodeDistanceFromSourceNode - 5, node.y! + addNodeDistanceFromSourceNode)
+    //   ctx.moveTo(node.x! + addNodeDistanceFromSourceNode - 5, node.y! + addNodeDistanceFromSourceNode)
-      ctx.lineTo(node.x! + addNodeDistanceFromSourceNode - 5 + 10, node.y! + addNodeDistanceFromSourceNode);
+    //   ctx.lineTo(node.x! + addNodeDistanceFromSourceNode - 5 + 10, node.y! + addNodeDistanceFromSourceNode);
-      ctx.stroke();
+    //   ctx.stroke();
 
-      ctx.beginPath();
+    //   ctx.beginPath();
-      ctx.moveTo(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode - 5)
+    //   ctx.moveTo(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode - 5)
-      ctx.lineTo(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode - 5 + 10);
+    //   ctx.lineTo(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode - 5 + 10);
-      ctx.stroke();
+    //   ctx.stroke();
-    }
+    // }
 
     // ctx.beginPath();
     // ctx.arc(node.x, node.y, nodeSize, 0, 2 * Math.PI);
@@ -179,9 +182,7 @@ export default function GraphView() {
     ctx.restore();
   }
 
-  function handleDagError(loopNodeIds: (string | number)[]) {
+  function handleDagError(loopNodeIds: (string | number)[]) {}
-    console.log(loopNodeIds);
-  }
 
   useEffect(() => {
     // add collision force
@@ -211,7 +212,7 @@ export default function GraphView() {
               nodeRelSize={nodeSize}
               nodeCanvasObject={renderNode}
               nodeCanvasObjectMode={() => "after"}
-              nodeAutoColorBy="group"
+              nodeAutoColorBy="type"
 
               linkLabel="label"
               linkCanvasObject={renderLink}
@@ -237,7 +238,7 @@ export default function GraphView() {
               nodeRelSize={20}
               nodeCanvasObject={renderNode}
               nodeCanvasObjectMode={() => "after"}
-              nodeAutoColorBy="group"
+              nodeAutoColorBy="type"
 
               linkLabel="label"
               linkCanvasObject={renderLink}
@@ -250,10 +251,10 @@ export default function GraphView() {
 
         <div className="absolute top-2 left-2 bg-gray-500 pt-4 pr-4 pb-4 pl-4 rounded-md max-w-2xl">
           <CogneeAddWidget onData={onDataChange} />
-          <CrewAITrigger />
+          <CrewAITrigger onData={onDataChange} onActivity={onActivityChange} />
         </div>
 
-        <div className="absolute top-2 right-2 bg-gray-500 pt-4 pr-4 pb-4 pl-4 rounded-md">
+        <div className="absolute top-2 right-2 bg-gray-500 pt-4 pr-4 pb-4 pl-4 rounded-md w-110">
           <GraphControls
             ref={graphControls}
             isAddNodeFormOpen={isAddNodeFormOpen}
@@ -265,10 +266,12 @@ export default function GraphView() {
       <Divider />
       <div className="pl-6 pr-6">
         <Footer>
-          <div className="flex flex-row items-center gap-6">
+          {(data?.nodes.length || data?.links.length) && (
-            <span>Nodes: {data?.nodes.length}</span>
+            <div className="flex flex-row items-center gap-6">
-            <span>Edges: {data?.links.length}</span>
+              <span>Nodes: {data?.nodes.length || 0}</span>
-          </div>
+              <span>Edges: {data?.links.length || 0}</span>
+            </div>
+          )}
         </Footer>
       </div>
     </main>

cognee-frontend/src/modules/ingestion/useDatasets.ts

@@ -42,7 +42,7 @@ function useDatasets() {
 
     statusTimeout.current = setTimeout(() => {
       checkDatasetStatuses(datasets);
-    }, 5000);
+    }, 50000);
   }, [fetchDatasetStatuses]);
   
   useEffect(() => {

cognee-frontend/src/ui/Partials/FeedbackForm.tsx

@@ -26,19 +26,22 @@ export default function FeedbackForm({ onSuccess }: FeedbackFormProps) {
     event.preventDefault();
     const formElements = event.currentTarget;
 
-    const authCredentials = new FormData();
-    authCredentials.append("feedback", formElements.feedback.value);
-
     setFeedbackError(null);
     disableFeedbackSubmit();
 
-    fetch("/v1/feedback/reasoning", {
+    fetch("/v1/crewai/feedback", {
       method: "POST",
-      body: authCredentials,
+      body: JSON.stringify({
+        feedback: formElements.feedback.value,
+      }),
+      headers: {
+        "Content-Type": "application/json",
+      },
     })
       .then(response => response.json())
       .then(() => {
         onSuccess();
+        formElements.feedback.value = "";
       })
       .catch(error => setFeedbackError(error.detail))
       .finally(() => enableFeedbackSubmit());
@@ -48,7 +51,7 @@ export default function FeedbackForm({ onSuccess }: FeedbackFormProps) {
     <form onSubmit={signIn} className="flex flex-col gap-2">
       <div className="flex flex-col gap-2">
         <div className="mb-4">
-          <label className="block text-white" htmlFor="feedback">Your feedback on agents reasoning</label>
+          <label className="block text-white" htmlFor="feedback">Feedback on agent&apos;s reasoning</label>
           <TextArea id="feedback" name="feedback" type="text" placeholder="Your feedback" />
         </div>
       </div>

cognee/api/client.py

@@ -47,9 +47,10 @@ app_environment = os.getenv("ENV", "prod")
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    # from cognee.modules.data.deletion import prune_system, prune_data
+    from cognee.modules.data.deletion import prune_system, prune_data
-    # await prune_data()
+
-    # await prune_system(metadata = True)
+    await prune_data()
+    await prune_system(metadata=True)
     # if app_environment == "local" or app_environment == "dev":
     from cognee.infrastructure.databases.relational import get_relational_engine
 
@@ -189,7 +190,7 @@ def start_api_server(host: str = "0.0.0.0", port: int = 8000):
     try:
         logger.info("Starting server at %s:%s", host, port)
 
-        uvicorn.run(app, host=host, port=port)
+        uvicorn.run(app, host=host, port=port, loop="asyncio")
     except Exception as e:
         logger.exception(f"Failed to start server: {e}")
         # Here you could add any cleanup code or error recovery code.

cognee/api/v1/cognify/cognify.py

@@ -2,17 +2,19 @@ import asyncio
 from pydantic import BaseModel
 from typing import Union, Optional
 
+from cognee.modules.users.methods import get_default_user
 from cognee.shared.logging_utils import get_logger
 from cognee.shared.data_models import KnowledgeGraph
 from cognee.infrastructure.llm import get_max_chunk_tokens
-
+from cognee.modules.users.models import User
 from cognee.modules.pipelines import cognee_pipeline
 from cognee.modules.pipelines.tasks.task import Task
 from cognee.modules.chunking.TextChunker import TextChunker
 from cognee.modules.ontology.rdf_xml.OntologyResolver import OntologyResolver
-from cognee.modules.pipelines.models.PipelineRunInfo import PipelineRunCompleted
+from cognee.modules.pipelines.models.PipelineRunInfo import PipelineRunCompleted, PipelineRunStarted
 from cognee.modules.pipelines.queues.pipeline_run_info_queues import push_to_queue
-from cognee.modules.users.models import User
+from cognee.modules.graph.operations import get_formatted_graph_data
+from cognee.modules.crewai.get_crewai_pipeline_run_id import get_crewai_pipeline_run_id
 
 from cognee.tasks.documents import (
     check_permissions_on_documents,
@@ -36,16 +38,20 @@ async def cognify(
     chunk_size: int = None,
     ontology_file_path: Optional[str] = None,
     run_in_background: bool = False,
+    is_stream_info_enabled: bool = False,
 ):
     tasks = await get_default_tasks(user, graph_model, chunker, chunk_size, ontology_file_path)
 
+    if not user:
+        user = await get_default_user()
+
     if run_in_background:
         return await run_cognify_as_background_process(tasks, user, datasets)
     else:
-        return await run_cognify_blocking(tasks, user, datasets)
+        return await run_cognify_blocking(tasks, user, datasets, is_stream_info_enabled)
 
 
-async def run_cognify_blocking(tasks, user, datasets):
+async def run_cognify_blocking(tasks, user, datasets, is_stream_info_enabled=False):
     pipeline_run_info = None
 
     async for run_info in cognee_pipeline(
@@ -53,6 +59,15 @@ async def run_cognify_blocking(tasks, user, datasets):
     ):
         pipeline_run_info = run_info
 
+        if (
+            is_stream_info_enabled
+            and not isinstance(pipeline_run_info, PipelineRunStarted)
+            and not isinstance(pipeline_run_info, PipelineRunCompleted)
+        ):
+            pipeline_run_id = get_crewai_pipeline_run_id(user.id)
+            pipeline_run_info.payload = await get_formatted_graph_data()
+            push_to_queue(pipeline_run_id, pipeline_run_info)
+
     return pipeline_run_info
 
 
@@ -68,6 +83,8 @@ async def run_cognify_as_background_process(tasks, user, datasets):
             try:
                 pipeline_run_info = await anext(pipeline_run)
 
+                pipeline_run_info.payload = await get_formatted_graph_data()
+
                 push_to_queue(pipeline_run_info.pipeline_run_id, pipeline_run_info)
 
                 if isinstance(pipeline_run_info, PipelineRunCompleted):

cognee/api/v1/cognify/routers/get_cognify_router.py

@@ -69,11 +69,15 @@ def get_cognify_router() -> APIRouter:
                 continue
 
             try:
-                await websocket.send_json({
+                await websocket.send_json(
-                    "pipeline_run_id": str(pipeline_run_info.pipeline_run_id),
+                    {
-                    "status": pipeline_run_info.status,
+                        "pipeline_run_id": str(pipeline_run_info.pipeline_run_id),
-                    "payload": await get_nodes_and_edges(pipeline_run_info.payload) if pipeline_run_info.payload else None,
+                        "status": pipeline_run_info.status,
-                })
+                        "payload": await get_nodes_and_edges(pipeline_run_info.payload)
+                        if pipeline_run_info.payload
+                        else None,
+                    }
+                )
 
                 if isinstance(pipeline_run_info, PipelineRunCompleted):
                     remove_queue(pipeline_run_id)
@@ -85,6 +89,7 @@ def get_cognify_router() -> APIRouter:
 
     return router
 
+
 async def get_nodes_and_edges(data_points):
     nodes = []
     edges = []
@@ -112,14 +117,24 @@ async def get_nodes_and_edges(data_points):
     nodes, edges = deduplicate_nodes_and_edges(nodes, edges)
 
     return {
-        "nodes": list(map(lambda node: {
+        "nodes": list(
-            "id": str(node.id),
+            map(
-            "label": node.name if hasattr(node, "name") else f"{node.type}_{str(node.id)}",
+                lambda node: {
-            "properties": {},
+                    "id": str(node.id),
-        }, nodes)),
+                    "label": node.name if hasattr(node, "name") else f"{node.type}_{str(node.id)}",
-        "edges": list(map(lambda edge: {
+                    "properties": {},
-            "source": str(edge[0]),
+                },
-            "target": str(edge[1]),
+                nodes,
-            "label": edge[2],
+            )
-        }, edges)),
+        ),
+        "edges": list(
+            map(
+                lambda edge: {
+                    "source": str(edge[0]),
+                    "target": str(edge[1]),
+                    "label": edge[2],
+                },
+                edges,
+            )
+        ),
     }

cognee/api/v1/crewai/routers/get_crewai_router.py

@@ -1,23 +1,117 @@
-from fastapi import APIRouter, Depends
+import os
+import asyncio
+from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect
+from starlette.status import WS_1000_NORMAL_CLOSURE, WS_1008_POLICY_VIOLATION
+
 from cognee.api.DTO import InDTO
-from cognee.modules.users.get_fastapi_users import get_fastapi_users
+from cognee.complex_demos.crewai_demo.src.crewai_demo.github_ingest_datapoints import (
-from cognee.modules.users.authentication.get_auth_backend import get_auth_backend
+    cognify_github_data_from_username,
-from cognee.modules.users.methods import get_authenticated_user
+)
+from cognee.modules.crewai.get_crewai_pipeline_run_id import get_crewai_pipeline_run_id
 from cognee.modules.users.models import User
+from cognee.modules.users.methods import get_authenticated_user
+from cognee.modules.pipelines.models import PipelineRunInfo, PipelineRunCompleted
+from cognee.complex_demos.crewai_demo.src.crewai_demo.main import (
+    # run_github_ingestion,
+    run_hiring_crew,
+)
+from cognee.modules.pipelines.queues.pipeline_run_info_queues import (
+    get_from_queue,
+    initialize_queue,
+    remove_queue,
+)
 
 
 class CrewAIRunPayloadDTO(InDTO):
     username1: str
     username2: str
 
+
+class CrewAIFeedbackPayloadDTO(InDTO):
+    feedback: str
+
+
 def get_crewai_router() -> APIRouter:
     router = APIRouter()
 
-    @router.post("/run", response_model=str)
+    @router.post("/run", response_model=bool)
-    async def run_crewai(payload: CrewAIRunPayloadDTO, user: User = Depends(get_authenticated_user)):
+    async def run_crewai(
+        payload: CrewAIRunPayloadDTO,
+        user: User = Depends(get_authenticated_user),
+    ):
         # Run CrewAI with the provided usernames
-        print(payload.username1, payload.username2)
+        # run_future = run_github_ingestion(payload.username1, payload.username2)
+        token = os.getenv("GITHUB_TOKEN")
 
-        return "CrewAI run started"
+        await cognify_github_data_from_username(payload.username1, token)
+        await cognify_github_data_from_username(payload.username2, token)
+
+        applicants = {
+            "applicant_1": payload.username1,
+            "applicant_2": payload.username2,
+        }
+
+        run_hiring_crew(applicants=applicants, number_of_rounds=2)
+
+        return True
+
+    @router.post("/feedback", response_model=None)
+    async def send_feedback(
+        payload: CrewAIFeedbackPayloadDTO,
+        user: User = Depends(
+            get_authenticated_user,
+        ),
+    ):
+        from cognee import add, cognify
+        # from secrets import choice
+        # from string import ascii_letters, digits
+
+        # hash6 = "".join(choice(ascii_letters + digits) for _ in range(6))
+        dataset_name = "final_reports"
+        await add(payload.feedback, node_set=["final_report"], dataset_name=dataset_name)
+        await cognify(datasets=dataset_name, is_stream_info_enabled=True)
+
+    @router.websocket("/subscribe")
+    async def subscribe_to_crewai_info(websocket: WebSocket):
+        await websocket.accept()
+
+        auth_message = await websocket.receive_json()
+
+        try:
+            user = await get_authenticated_user(auth_message.get("Authorization"))
+        except Exception:
+            await websocket.close(code=WS_1008_POLICY_VIOLATION, reason="Unauthorized")
+            return
+
+        pipeline_run_id = get_crewai_pipeline_run_id(user.id)
+
+        initialize_queue(pipeline_run_id)
+
+        while True:
+            pipeline_run_info = get_from_queue(pipeline_run_id)
+
+            if not pipeline_run_info:
+                await asyncio.sleep(2)
+                continue
+
+            if not isinstance(pipeline_run_info, PipelineRunInfo):
+                continue
+
+            try:
+                await websocket.send_json(
+                    {
+                        "pipeline_run_id": str(pipeline_run_info.pipeline_run_id),
+                        "status": pipeline_run_info.status,
+                        "payload": pipeline_run_info.payload if pipeline_run_info.payload else None,
+                    }
+                )
+
+                if isinstance(pipeline_run_info, PipelineRunCompleted):
+                    remove_queue(pipeline_run_id)
+                    await websocket.close(code=WS_1000_NORMAL_CLOSURE)
+                    break
+            except WebSocketDisconnect:
+                remove_queue(pipeline_run_id)
+                break
 
     return router

cognee/api/v1/datasets/routers/get_datasets_router.py

@@ -1,3 +1,4 @@
+from cognee.modules.graph.operations import get_formatted_graph_data
 from cognee.shared.logging_utils import get_logger
 from fastapi import APIRouter
 from datetime import datetime
@@ -44,11 +45,13 @@ class GraphNodeDTO(OutDTO):
     label: str
     properties: dict
 
+
 class GraphEdgeDTO(OutDTO):
     source: UUID
     target: UUID
     label: str
 
+
 class GraphDTO(OutDTO):
     nodes: List[GraphNodeDTO]
     edges: List[GraphEdgeDTO]
@@ -111,26 +114,10 @@ def get_datasets_router() -> APIRouter:
 
     @router.get("/{dataset_id}/graph", response_model=GraphDTO)
     async def get_dataset_graph(dataset_id: UUID, user: User = Depends(get_authenticated_user)):
-        from cognee.infrastructure.databases.graph import get_graph_engine
-
         try:
-            graph_client = await get_graph_engine()
-            (nodes, edges) = await graph_client.get_graph_data()
-
             return JSONResponse(
                 status_code=200,
-                content={
+                content=await get_formatted_graph_data(),
-                    "nodes": list(map(lambda node: {
-                        "id": str(node[0]),
-                        "label": node[1]["name"] if hasattr(node[1], "name") else f"{node[1]['type']}_{str(node[0])}",
-                        "properties": {},
-                    }, nodes)),
-                    "edges": list(map(lambda edge: {
-                        "source": str(edge[0]),
-                        "target": str(edge[1]),
-                        "label": edge[2],
-                    }, edges)),
-                },
             )
         except Exception as error:
             print(error)

cognee/complex_demos/crewai_demo/src/crewai_demo/custom_tools/cognee_build.py

@@ -19,7 +19,7 @@ class CogneeBuild(BaseTool):
                     node_set = meta["nodeset"]
                     await cognee.add(text, node_set=node_set)
 
-                await cognee.cognify()
+                await cognee.cognify(is_stream_info_enabled=True)
 
                 return "Knowledge Graph is done."
             except Exception as e:
@@ -27,10 +27,11 @@ class CogneeBuild(BaseTool):
 
         try:
             loop = asyncio.get_event_loop()
-            if loop.is_running():
+
+            if not loop.is_running():
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
-            results = loop.run_until_complete(main())
+
-            return results
+            return loop.create_task(main())
         except Exception as e:
             return f"Tool execution error: {str(e)}"

cognee/complex_demos/crewai_demo/src/crewai_demo/custom_tools/cognee_ingestion.py

@@ -1,8 +1,8 @@
-from crewai.tools import BaseTool
-from typing import Type, List
-from pydantic import BaseModel, Field, PrivateAttr
-from cognee.modules.engine.models import NodeSet
 import asyncio
+import nest_asyncio
+from crewai.tools import BaseTool
+from typing import Type
+from pydantic import BaseModel, Field
 
 
 class CogneeIngestionInput(BaseModel):
@@ -24,14 +24,15 @@ class CogneeIngestion(BaseTool):
 
     def _run(self, text: str) -> str:
         import cognee
-        from secrets import choice
+        # from secrets import choice
-        from string import ascii_letters, digits
+        # from string import ascii_letters, digits
 
         async def main():
             try:
-                hash6 = "".join(choice(ascii_letters + digits) for _ in range(6))
+                # hash6 = "".join(choice(ascii_letters + digits) for _ in range(6))
-                await cognee.add(text, node_set=[self._nodeset_name], dataset_name=hash6)
+                dataset_name = "final_reports"
-                await cognee.cognify(datasets=hash6)
+                await cognee.add(text, node_set=[self._nodeset_name], dataset_name=dataset_name)
+                await cognee.cognify(datasets=dataset_name, is_stream_info_enabled=True)
 
                 return "Report ingested successfully into Cognee memory."
             except Exception as e:
@@ -39,9 +40,15 @@ class CogneeIngestion(BaseTool):
 
         try:
             loop = asyncio.get_event_loop()
-            if loop.is_running():
+
+            if not loop.is_running():
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
-            return loop.run_until_complete(main())
+
+            nest_asyncio.apply(loop)
+
+            result = loop.run_until_complete(main())
+
+            return result
         except Exception as e:
             return f"Tool execution error: {str(e)}"

cognee/complex_demos/crewai_demo/src/crewai_demo/custom_tools/cognee_search.py

@@ -1,3 +1,5 @@
+import nest_asyncio
+
 from crewai.tools import BaseTool
 from typing import Type
 from pydantic import BaseModel, Field, PrivateAttr
@@ -47,10 +49,15 @@ class CogneeSearch(BaseTool):
 
         try:
             loop = asyncio.get_event_loop()
-            if loop.is_running():
+
+            if not loop.is_running():
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
-            search_results = loop.run_until_complete(main())
+
-            return search_results
+            nest_asyncio.apply(loop)
+
+            result = loop.run_until_complete(main())
+
+            return result
         except Exception as e:
             return f"Tool execution error: {str(e)}"

cognee/complex_demos/crewai_demo/src/crewai_demo/custom_tools/github_ingestion.py

@@ -1,7 +1,5 @@
 from crewai.tools import BaseTool
 
-from cognee.modules.engine.models import NodeSet
-from cognee.modules.retrieval.graph_completion_retriever import GraphCompletionRetriever
 from ..github_ingest_datapoints import cognify_github_data_from_username
 
 
@@ -11,30 +9,32 @@ class GithubIngestion(BaseTool):
 
     def _run(self, applicant_1, applicant_2) -> str:
         import asyncio
-        import cognee
+
+        # import cognee
         import os
-        from cognee.low_level import DataPoint, setup as cognee_setup
+        # from cognee.low_level import setup as cognee_setup
 
         async def main():
             try:
-                await cognee.prune.prune_data()
+                # await cognee.prune.prune_data()
-                await cognee.prune.prune_system(metadata=True)
+                # await cognee.prune.prune_system(metadata=True)
-                await cognee_setup()
+                # await cognee_setup()
                 token = os.getenv("GITHUB_TOKEN")
 
                 await cognify_github_data_from_username(applicant_1, token)
                 await cognify_github_data_from_username(applicant_2, token)
 
-                return "Github ingestion finished"
+                return True
             except Exception as e:
                 return f"Error: {str(e)}"
 
         try:
             loop = asyncio.get_event_loop()
-            if loop.is_running():
+
+            if not loop.is_running():
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
-            results = loop.run_until_complete(main())
+
-            return results
+            return loop.create_task(main())
         except Exception as e:
             return f"Tool execution error: {str(e)}"

cognee/complex_demos/crewai_demo/src/crewai_demo/github_datapoint_creators.py

@@ -1,7 +1,6 @@
 from uuid import uuid5, NAMESPACE_OID
-from typing import Dict, Any, List, Tuple, Optional
+from typing import Dict, Any, List
 
-from cognee.low_level import DataPoint
 from cognee.modules.engine.models.node_set import NodeSet
 from cognee.shared.logging_utils import get_logger
 from cognee.complex_demos.crewai_demo.src.crewai_demo.github_datapoints import (
@@ -60,7 +59,9 @@ def create_file_datapoint(filename: str, repo_name: str, nodesets: List[NodeSet]
     """Creates a File DataPoint with a consistent ID."""
     file_key = f"{repo_name}:{filename}"
     file_id = uuid5(NAMESPACE_OID, file_key)
-    file = File(id=file_id, filename=filename, repo=repo_name, belongs_to_set=nodesets)
+    file = File(
+        id=file_id, name=filename, filename=filename, repo=repo_name, belongs_to_set=nodesets
+    )
     logger.debug(f"Created File with ID: {file_id} for {filename}")
     return file
 
@@ -72,6 +73,7 @@ def create_commit_datapoint(
     commit_id = uuid5(NAMESPACE_OID, commit_data.get("commit_sha", ""))
     commit = Commit(
         id=commit_id,
+        name=commit_data.get("commit_sha", ""),
         commit_sha=commit_data.get("commit_sha", ""),
         text="Commit message:" + (str)(commit_data.get("commit_message", "")),
         commit_date=commit_data.get("commit_date", ""),
@@ -96,6 +98,7 @@ def create_file_change_datapoint(
 
     file_change = FileChange(
         id=fc_id,
+        name=fc_data.get("filename", ""),
         filename=fc_data.get("filename", ""),
         status=fc_data.get("status", ""),
         additions=fc_data.get("additions", 0),
@@ -121,6 +124,7 @@ def create_issue_datapoint(
 
     issue = Issue(
         id=issue_id,
+        name=str(issue_data.get("issue_number", 0)),
         number=issue_data.get("issue_number", 0),
         text=issue_data.get("issue_title", ""),
         state=issue_data.get("issue_state", ""),
@@ -142,6 +146,7 @@ def create_comment_datapoint(
 
     comment = Comment(
         id=comment_id,
+        name=str(comment_data.get("comment_id", "")),
         comment_id=str(comment_data.get("comment_id", "")),
         text=comment_data.get("body", ""),
         created_at=comment_data.get("created_at", ""),

cognee/complex_demos/crewai_demo/src/crewai_demo/github_datapoints.py

@@ -7,6 +7,7 @@ class File(DataPoint):
     """File is now a leaf node without any lists of other DataPoints"""
 
     filename: str
+    name: str
     repo: str
     metadata: dict = {"index_fields": ["filename"]}
 
@@ -25,6 +26,7 @@ class GitHubUser(DataPoint):
 
 class FileChange(DataPoint):
     filename: str
+    name: str
     status: str
     additions: int
     deletions: int
@@ -39,6 +41,7 @@ class FileChange(DataPoint):
 
 class Comment(DataPoint):
     comment_id: str
+    name: str
     text: str
     created_at: str
     updated_at: str
@@ -51,6 +54,7 @@ class Comment(DataPoint):
 
 class Issue(DataPoint):
     number: int
+    name: str
     text: str
     state: str
     repository: str
@@ -60,6 +64,7 @@ class Issue(DataPoint):
 
 class Commit(DataPoint):
     commit_sha: str
+    name: str
     text: str
     commit_date: str
     commit_url: str

cognee/complex_demos/crewai_demo/src/crewai_demo/github_ingest_datapoints.py

@@ -2,11 +2,8 @@ import json
 import asyncio
 from uuid import uuid5, NAMESPACE_OID
 from typing import Optional, List, Dict, Any
-from pathlib import Path
-from cognee.api.v1.search import SearchType
 import cognee
-from cognee.low_level import DataPoint, setup as cognee_setup
+from cognee.low_level import DataPoint
-from cognee.modules.retrieval.graph_completion_retriever import GraphCompletionRetriever
 from cognee.tasks.storage import add_data_points
 from cognee.modules.pipelines.tasks.task import Task
 from cognee.modules.pipelines import run_tasks
@@ -16,6 +13,9 @@ from cognee.shared.logging_utils import get_logger
 from cognee.complex_demos.crewai_demo.src.crewai_demo.github_ingest import (
     get_github_data_for_cognee,
 )
+from cognee.modules.pipelines.models.PipelineRunInfo import PipelineRunCompleted, PipelineRunStarted
+from cognee.modules.graph.operations import get_formatted_graph_data
+from cognee.modules.crewai.get_crewai_pipeline_run_id import get_crewai_pipeline_run_id
 
 # Import DataPoint classes from github_datapoints.py
 from cognee.complex_demos.crewai_demo.src.crewai_demo.github_datapoints import (
@@ -205,6 +205,27 @@ def build_github_datapoints_from_dict(github_data: Dict[str, Any]):
     return all_datapoints
 
 
+async def run_with_info_stream(tasks, user, data, dataset_id, pipeline_name):
+    from cognee.modules.pipelines.queues.pipeline_run_info_queues import push_to_queue
+
+    pipeline_run = run_tasks(
+        tasks=tasks,
+        data=data,
+        dataset_id=dataset_id,
+        pipeline_name=pipeline_name,
+        user=user,
+    )
+
+    pipeline_run_id = get_crewai_pipeline_run_id(user.id)
+
+    async for pipeline_run_info in pipeline_run:
+        if not isinstance(pipeline_run_info, PipelineRunStarted) and not isinstance(
+            pipeline_run_info, PipelineRunCompleted
+        ):
+            pipeline_run_info.payload = await get_formatted_graph_data()
+            push_to_queue(pipeline_run_id, pipeline_run_info)
+
+
 async def cognify_github_data(github_data: dict):
     """Process GitHub user, file changes, and comments data from a loaded dictionary."""
     all_datapoints = build_github_datapoints_from_dict(github_data)
@@ -216,18 +237,16 @@ async def cognify_github_data(github_data: dict):
 
     cognee_user = await get_default_user()
     tasks = [Task(add_data_points, task_config={"batch_size": 50})]
-    results = run_tasks(
+
+    await run_with_info_stream(
         tasks=tasks,
         data=all_datapoints,
         dataset_id=dataset_id,
         pipeline_name="github_pipeline",
         user=cognee_user,
     )
-    async for result in results:
-        print(result)
 
     logger.info(f"Done processing {len(all_datapoints)} datapoints")
-    return True
 
 
 async def cognify_github_data_from_username(
@@ -263,8 +282,6 @@ async def cognify_github_data_from_username(
 
     await cognify_github_data(github_data)
 
-    return None
-
 
 async def process_github_from_file(json_file_path: str):
     """Process GitHub data from a JSON file."""

cognee/complex_demos/crewai_demo/src/crewai_demo/hiring_crew.py

@@ -1,6 +1,5 @@
-import os
 from crewai import Agent, Crew, Process, Task
-from crewai.project import CrewBase, agent, crew, task, before_kickoff
+from crewai.project import CrewBase, agent, crew, task
 from pydantic import BaseModel
 
 from cognee.complex_demos.crewai_demo.src.crewai_demo.custom_tools.cognee_ingestion import (

cognee/complex_demos/crewai_demo/src/crewai_demo/main.py

@@ -1,6 +1,6 @@
 import warnings
 import os
-from hiring_crew import HiringCrew
+from .hiring_crew import HiringCrew
 from cognee.complex_demos.crewai_demo.src.crewai_demo.custom_tools.github_ingestion import (
     GithubIngestion,
 )
@@ -14,7 +14,7 @@ def print_environment():
 
 
 def run_github_ingestion(applicant_1, applicant_2):
-    GithubIngestion().run(applicant_1=applicant_1, applicant_2=applicant_2)
+    return GithubIngestion().run(applicant_1=applicant_1, applicant_2=applicant_2)
 
 
 def run_hiring_crew(applicants: dict, number_of_rounds: int = 1, llm_client=None):

cognee/infrastructure/llm/openai/adapter.py

@@ -50,7 +50,7 @@ class OpenAIAdapter(LLMInterface):
         self.max_tokens = max_tokens
         self.streaming = streaming
 
-    @observe(as_type="generation")
+    # @observe(as_type="generation")
     @sleep_and_retry_async()
     @rate_limit_async
     async def acreate_structured_output(
@@ -77,7 +77,7 @@ class OpenAIAdapter(LLMInterface):
             max_retries=self.MAX_RETRIES,
         )
 
-    @observe
+    # @observe
     @sleep_and_retry_sync()
     @rate_limit_sync
     def create_structured_output(

cognee/modules/crewai/get_crewai_pipeline_run_id.py

@@ -0,0 +1,11 @@
+from uuid import NAMESPACE_OID, UUID, uuid5
+
+from cognee.modules.pipelines.utils import generate_pipeline_id, generate_pipeline_run_id
+
+
+def get_crewai_pipeline_run_id(user_id: UUID):
+    dataset_id = uuid5(NAMESPACE_OID, "GitHub")
+    pipeline_id = generate_pipeline_id(user_id, "github_pipeline")
+    pipeline_run_id = generate_pipeline_run_id(pipeline_id, dataset_id)
+
+    return pipeline_run_id

cognee/modules/graph/operations/__init__.py

@@ -0,0 +1 @@
+from .get_formatted_graph_data import get_formatted_graph_data

cognee/modules/graph/operations/get_formatted_graph_data.py

@@ -0,0 +1,36 @@
+from cognee.infrastructure.databases.graph import get_graph_engine
+
+
+async def get_formatted_graph_data():
+    graph_client = await get_graph_engine()
+    (nodes, edges) = await graph_client.get_graph_data()
+
+    return {
+        "nodes": list(
+            map(
+                lambda node: {
+                    "id": str(node[0]),
+                    "label": node[1]["name"]
+                    if ("name" in node[1] and node[1]["name"] != "")
+                    else f"{node[1]['type']}_{str(node[0])}",
+                    "type": node[1]["type"],
+                    "properties": {
+                        key: value
+                        for key, value in node[1].items()
+                        if key not in ["id", "type", "name"] and value is not None
+                    },
+                },
+                nodes,
+            )
+        ),
+        "edges": list(
+            map(
+                lambda edge: {
+                    "source": str(edge[0]),
+                    "target": str(edge[1]),
+                    "label": edge[2],
+                },
+                edges,
+            )
+        ),
+    }

cognee/modules/pipelines/models/PipelineRunInfo.py

@@ -15,19 +15,19 @@ class PipelineRunInfo(BaseModel):
 
 class PipelineRunStarted(PipelineRunInfo):
     status: str = "PipelineRunStarted"
-    pass
 
 
 class PipelineRunYield(PipelineRunInfo):
     status: str = "PipelineRunYield"
-    pass
 
 
 class PipelineRunCompleted(PipelineRunInfo):
     status: str = "PipelineRunCompleted"
-    pass
 
 
 class PipelineRunErrored(PipelineRunInfo):
     status: str = "PipelineRunErrored"
-    pass
+
+
+class PipelineRunActivity(BaseModel):
+    status: str = "PipelineRunActivity"

pyproject.toml

@@ -6,7 +6,7 @@ authors = [
     { name = "Vasilije Markovic" },
     { name = "Boris Arzentar" },
 ]
-requires-python = ">=3.10,<=3.13"
+requires-python = ">=3.10,<3.13"
 readme = "README.md"
 license = "Apache-2.0"
 classifiers = [
@@ -58,6 +58,7 @@ dependencies = [
     "structlog>=25.2.0,<26",
     "onnxruntime<=1.21.1",
     "pylance==0.22.0",
+    "nest-asyncio>=1.6.0",
 ]
 
 [project.optional-dependencies]
@@ -91,10 +92,6 @@ falkordb = ["falkordb==1.0.9"]
 kuzu = ["kuzu==0.8.2"]
 groq = ["groq==0.8.0"]
 milvus = ["pymilvus>=2.5.0,<3"]
-chromadb = [
-    "chromadb>=0.3.0,<0.7",
-    "pypika==0.48.8",
-]
 docs = ["unstructured[csv, doc, docx, epub, md, odt, org, ppt, pptx, rst, rtf, tsv, xlsx]>=0.16.13,<0.17"]
 codegraph = [
     "fastembed<=0.6.0 ; python_version < '3.13'",
@@ -128,6 +125,10 @@ dev = [
     "mkdocstrings[python]>=0.26.2,<0.27",
 ]
 debug = ["debugpy==1.8.9"]
+crewai = [
+    "crewai>=0.117.1",
+    "pygithub>=2.6.1",
+]
 
 [project.urls]
 Homepage = "https://www.cognee.ai"