feat: enable feeeback and activity log
This commit is contained in:
Boris Arzentar
parent
6da557565a
commit
fac930dc59
27 changed files with 969 additions and 1294 deletions
|
|
@ -1,5 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import { v4 as uuid4 } from "uuid";
|
||||
import { ChangeEvent, useEffect } from "react";
|
||||
import { CTAButton, StatusIndicator } from "@/ui/elements";
|
||||
|
||||
|
|
@ -30,15 +31,16 @@ export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
|
|||
.then((datasets) => {
|
||||
const dataset = datasets?.[0];
|
||||
|
||||
if (dataset) {
|
||||
getDatasetGraph(dataset)
|
||||
.then((graph) => onData({
|
||||
nodes: graph.nodes,
|
||||
links: graph.edges,
|
||||
}));
|
||||
}
|
||||
// For CrewAI we don't have a dataset.
|
||||
// if (dataset) {
|
||||
getDatasetGraph(dataset || { id: uuid4() })
|
||||
.then((graph) => onData({
|
||||
nodes: graph.nodes,
|
||||
links: graph.edges,
|
||||
}));
|
||||
// }
|
||||
});
|
||||
}, [refreshDatasets]);
|
||||
}, [onData, refreshDatasets]);
|
||||
|
||||
const handleAddFiles = (dataset: { id?: string, name?: string }, event: ChangeEvent<HTMLInputElement>) => {
|
||||
event.stopPropagation();
|
||||
|
|
@ -51,8 +53,6 @@ export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
|
|||
|
||||
return addData(dataset, files)
|
||||
.then(() => {
|
||||
console.log("Data added successfully.");
|
||||
|
||||
const onUpdate = (data: any) => {
|
||||
onData({
|
||||
nodes: data.payload.nodes,
|
||||
|
|
@ -60,11 +60,12 @@ export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
|
|||
});
|
||||
};
|
||||
|
||||
return cognifyDataset(dataset, onUpdate)
|
||||
.then((data) => console.log(data));
|
||||
return cognifyDataset(dataset, onUpdate);
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
return null;
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-4 mb-4">
|
||||
{datasets.length ? datasets.map((dataset) => (
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
import { useState } from "react";
|
||||
import { fetch } from "@/utils";
|
||||
import { v4 as uuid4 } from "uuid";
|
||||
import { LoadingIndicator } from "@/ui/App";
|
||||
import { CTAButton, Input } from "@/ui/elements";
|
||||
|
||||
interface CrewAIFormPayload extends HTMLFormElement {
|
||||
|
|
@ -6,21 +9,98 @@ interface CrewAIFormPayload extends HTMLFormElement {
|
|||
username2: HTMLInputElement;
|
||||
}
|
||||
|
||||
export default function CrewAITrigger() {
|
||||
interface CrewAITriggerProps {
|
||||
onData: (data: any) => void;
|
||||
onActivity: (activities: any) => void;
|
||||
}
|
||||
|
||||
export default function CrewAITrigger({ onData, onActivity }: CrewAITriggerProps) {
|
||||
const [isCrewAIRunning, setIsCrewAIRunning] = useState(false);
|
||||
|
||||
const handleRunCrewAI = (event: React.FormEvent<CrewAIFormPayload>) => {
|
||||
fetch("/v1/crew-ai/run", {
|
||||
event.preventDefault();
|
||||
const formElements = event.currentTarget;
|
||||
|
||||
const crewAIConfig = {
|
||||
username1: formElements.username1.value,
|
||||
username2: formElements.username2.value,
|
||||
};
|
||||
|
||||
const websocket = new WebSocket("ws://localhost:8000/api/v1/crewai/subscribe");
|
||||
|
||||
websocket.onopen = () => {
|
||||
websocket.send(JSON.stringify({
|
||||
"Authorization": `Bearer ${localStorage.getItem("access_token")}`,
|
||||
}));
|
||||
};
|
||||
|
||||
let isCrewAIDone = false;
|
||||
onActivity([{ id: uuid4(), timestamp: Date.now(), activity: "Running CrewAI" }]);
|
||||
|
||||
websocket.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
|
||||
if (data.status === "PipelineRunActivity") {
|
||||
onActivity(data.payload);
|
||||
return;
|
||||
}
|
||||
|
||||
onData({
|
||||
nodes: data.payload.nodes,
|
||||
links: data.payload.edges,
|
||||
});
|
||||
|
||||
const nodes_type_map: { [key: string]: number } = {};
|
||||
|
||||
for (let i = 0; i < data.payload.nodes.length; i++) {
|
||||
const node = data.payload.nodes[i];
|
||||
if (!nodes_type_map[node.type]) {
|
||||
nodes_type_map[node.type] = 0;
|
||||
}
|
||||
nodes_type_map[node.type] += 1;
|
||||
}
|
||||
|
||||
const activityMessage = Object.entries(nodes_type_map).reduce((message, [type, count]) => {
|
||||
return `${message}\n | ${type}: ${count}`;
|
||||
}, "Graph updated:");
|
||||
|
||||
onActivity([{
|
||||
id: uuid4(),
|
||||
timestamp: Date.now(),
|
||||
activity: activityMessage,
|
||||
}]);
|
||||
|
||||
if (data.status === "PipelineRunCompleted") {
|
||||
isCrewAIDone = true;
|
||||
websocket.close();
|
||||
}
|
||||
};
|
||||
|
||||
setIsCrewAIRunning(true);
|
||||
|
||||
return fetch("/v1/crewai/run", {
|
||||
method: "POST",
|
||||
body: new FormData(event.currentTarget),
|
||||
body: JSON.stringify(crewAIConfig),
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then((data) => console.log(data));
|
||||
.finally(() => {
|
||||
websocket.close();
|
||||
setIsCrewAIRunning(false);
|
||||
onActivity([{ id: uuid4(), timestamp: Date.now(), activity: "CrewAI run done" }]);
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<form className="w-full flex flex-row gap-2 items-center" onSubmit={handleRunCrewAI}>
|
||||
<Input type="text" placeholder="Github Username" required />
|
||||
<Input type="text" placeholder="Github Username" required />
|
||||
<CTAButton type="submit" className="whitespace-nowrap">Run CrewAI</CTAButton>
|
||||
<Input name="username1" type="text" placeholder="Github Username" required defaultValue="hajdul88" />
|
||||
<Input name="username2" type="text" placeholder="Github Username" required defaultValue="lxobr" />
|
||||
<CTAButton type="submit" className="whitespace-nowrap">
|
||||
Run CrewAI
|
||||
{isCrewAIRunning && <LoadingIndicator />}
|
||||
</CTAButton>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,31 +19,38 @@ interface GraphControlsProps {
|
|||
export interface GraphControlsAPI {
|
||||
setSelectedNode: (node: NodeObject | null) => void;
|
||||
getSelectedNode: () => NodeObject | null;
|
||||
updateActivity: (activities: ActivityLog[]) => void;
|
||||
}
|
||||
|
||||
type ActivityLog = {
|
||||
id: string;
|
||||
timestamp: number;
|
||||
activity: string;
|
||||
}[];
|
||||
};
|
||||
|
||||
type NodeProperties = {
|
||||
type NodeProperty = {
|
||||
id: string;
|
||||
name: string;
|
||||
value: string;
|
||||
}[];
|
||||
};
|
||||
|
||||
const formatter = new Intl.DateTimeFormat("en-GB", { dateStyle: "short", timeStyle: "medium" });
|
||||
|
||||
export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, onFitIntoView, ref }: GraphControlsProps) {
|
||||
const [selectedNode, setSelectedNode] = useState<NodeObject | null>(null);
|
||||
const [activityLog, setActivityLog] = useState<ActivityLog>([]);
|
||||
const [nodeProperties, setNodeProperties] = useState<NodeProperties>([]);
|
||||
const [newProperty, setNewProperty] = useState<NodeProperties[0]>({
|
||||
const [activityLog, setActivityLog] = useState<ActivityLog[]>([]);
|
||||
const [nodeProperties, setNodeProperties] = useState<NodeProperty[]>([]);
|
||||
const [newProperty, setNewProperty] = useState<NodeProperty>({
|
||||
id: uuid4(),
|
||||
name: "",
|
||||
value: "",
|
||||
});
|
||||
|
||||
const handlePropertyChange = (property: NodeProperties[0], property_key: string, event: ChangeEvent<HTMLInputElement>) => {
|
||||
const updateActivity = (newActivities: ActivityLog[]) => {
|
||||
setActivityLog((activities) => [...activities, ...newActivities]);
|
||||
};
|
||||
|
||||
const handlePropertyChange = (property: NodeProperty, property_key: string, event: ChangeEvent<HTMLInputElement>) => {
|
||||
const value = event.target.value;
|
||||
|
||||
setNodeProperties(nodeProperties.map((nodeProperty) => (nodeProperty.id === property.id ? {...nodeProperty, [property_key]: value } : nodeProperty)));
|
||||
|
|
@ -58,11 +65,11 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
|
|||
}
|
||||
};
|
||||
|
||||
const handlePropertyDelete = (property: NodeProperties[0]) => {
|
||||
const handlePropertyDelete = (property: NodeProperty) => {
|
||||
setNodeProperties(nodeProperties.filter((nodeProperty) => nodeProperty.id !== property.id));
|
||||
};
|
||||
|
||||
const handleNewPropertyChange = (property: NodeProperties[0], property_key: string, event: ChangeEvent<HTMLInputElement>) => {
|
||||
const handleNewPropertyChange = (property: NodeProperty, property_key: string, event: ChangeEvent<HTMLInputElement>) => {
|
||||
const value = event.target.value;
|
||||
|
||||
setNewProperty({...property, [property_key]: value });
|
||||
|
|
@ -71,6 +78,7 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
|
|||
useImperativeHandle(ref, () => ({
|
||||
setSelectedNode,
|
||||
getSelectedNode: () => selectedNode,
|
||||
updateActivity,
|
||||
}));
|
||||
|
||||
const [selectedTab, setSelectedTab] = useState("nodeDetails");
|
||||
|
|
@ -81,14 +89,14 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
|
|||
|
||||
return (
|
||||
<>
|
||||
<div className="flex">
|
||||
<button onClick={() => setSelectedTab("nodeDetails")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30", { "border-b-indigo-600 text-white": selectedTab === "nodeDetails" })}>
|
||||
<div className="flex w-full">
|
||||
<button onClick={() => setSelectedTab("nodeDetails")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30 flex-1/3", { "border-b-indigo-600 text-white": selectedTab === "nodeDetails" })}>
|
||||
<span className="whitespace-nowrap">Node Details</span>
|
||||
</button>
|
||||
<button onClick={() => setSelectedTab("activityLog")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30", { "border-b-indigo-600 text-white": selectedTab === "activityLog" })}>
|
||||
<button onClick={() => setSelectedTab("activityLog")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30 flex-1/3", { "border-b-indigo-600 text-white": selectedTab === "activityLog" })}>
|
||||
<span className="whitespace-nowrap">Activity Log</span>
|
||||
</button>
|
||||
<button onClick={() => setSelectedTab("feedback")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30", { "border-b-indigo-600 text-white": selectedTab === "feedback" })}>
|
||||
<button onClick={() => setSelectedTab("feedback")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30 flex-1/3", { "border-b-indigo-600 text-white": selectedTab === "feedback" })}>
|
||||
<span className="whitespace-nowrap">Feedback</span>
|
||||
</button>
|
||||
</div>
|
||||
|
|
@ -97,9 +105,9 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
|
|||
{selectedTab === "nodeDetails" && (
|
||||
<>
|
||||
<div className="w-full flex flex-row gap-2 items-center mb-4">
|
||||
<label className="text-gray-300 whitespace-nowrap">Graph Shape:</label>
|
||||
<Select onChange={handleGraphShapeControl}>
|
||||
<option selected value="none">None</option>
|
||||
<label className="text-gray-300 whitespace-nowrap flex-1/5">Graph Shape:</label>
|
||||
<Select defaultValue="none" onChange={handleGraphShapeControl} className="flex-2/5">
|
||||
<option value="none">None</option>
|
||||
<option value="td">Top-down</option>
|
||||
<option value="bu">Bottom-up</option>
|
||||
<option value="lr">Left-right</option>
|
||||
|
|
@ -107,9 +115,9 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
|
|||
<option value="radialin">Radial-in</option>
|
||||
<option value="radialout">Radial-out</option>
|
||||
</Select>
|
||||
<NeutralButton onClick={onFitIntoView} className="flex-2/5 whitespace-nowrap">Fit Graph into View</NeutralButton>
|
||||
</div>
|
||||
|
||||
<NeutralButton onClick={onFitIntoView} className="mb-4">Fit Graph into View</NeutralButton>
|
||||
|
||||
{isAddNodeFormOpen ? (
|
||||
<form className="flex flex-col gap-4" onSubmit={() => {}}>
|
||||
|
|
@ -138,21 +146,25 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
|
|||
) : (
|
||||
selectedNode ? (
|
||||
<div className="flex flex-col gap-4">
|
||||
<div className="flex flex-col gap-2">
|
||||
<div className="flex gap-2 items-center">
|
||||
<div className="flex flex-col gap-2 overflow-y-auto max-h-96">
|
||||
<div className="flex gap-2 items-top">
|
||||
<span className="text-gray-300">ID:</span>
|
||||
<span className="text-white">{selectedNode.id}</span>
|
||||
</div>
|
||||
<div className="flex gap-2 items-top">
|
||||
<span className="text-gray-300">Label:</span>
|
||||
<span className="text-white">{selectedNode.label}</span>
|
||||
</div>
|
||||
|
||||
{Object.entries(selectedNode.properties).map(([key, value]) => (
|
||||
<div key={key} className="flex gap-2 items-center">
|
||||
<span className="text-gray-300">{key}:</span>
|
||||
<span className="text-white">{typeof value === "object" ? JSON.stringify(value) : value as string}</span>
|
||||
<div key={key} className="flex gap-2 items-top">
|
||||
<span className="text-gray-300">{key.charAt(0).toUpperCase() + key.slice(1)}:</span>
|
||||
<span className="text-white truncate">{typeof value === "object" ? JSON.stringify(value) : value as string}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<CTAButton type="button" onClick={() => {}}>Edit Node</CTAButton>
|
||||
{/* <CTAButton type="button" onClick={() => {}}>Edit Node</CTAButton> */}
|
||||
</div>
|
||||
) : (
|
||||
<span className="text-white">No node selected.</span>
|
||||
|
|
@ -164,9 +176,9 @@ export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, o
|
|||
{selectedTab === "activityLog" && (
|
||||
<div className="flex flex-col gap-2">
|
||||
{activityLog.map((activity) => (
|
||||
<div key={activity.id} className="flex gap-2 items-center">
|
||||
<span className="text-gray-300">{activity.timestamp}</span>
|
||||
<span className="text-white">{activity.activity}</span>
|
||||
<div key={activity.id} className="flex gap-2 items-top">
|
||||
<span className="text-gray-300 whitespace-nowrap">{formatter.format(activity.timestamp)}: </span>
|
||||
<span className="text-white whitespace-normal">{activity.activity}</span>
|
||||
</div>
|
||||
))}
|
||||
{!activityLog.length && <span className="text-white">No activity logged.</span>}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"use client";
|
||||
|
||||
import { forceCollide, forceManyBody } from "d3-force-3d";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import ForceGraph, { ForceGraphMethods, LinkObject, NodeObject } from "react-force-graph-2d";
|
||||
|
||||
import { TextLogo } from "@/ui/App";
|
||||
|
|
@ -35,24 +35,25 @@ export default function GraphView() {
|
|||
|
||||
const [data, updateData] = useState<GraphData | null>(null);
|
||||
|
||||
const onDataChange = (newData: NodesAndEdges) => {
|
||||
if (data === null) {
|
||||
updateData({
|
||||
nodes: newData.nodes,
|
||||
links: newData.links,
|
||||
});
|
||||
} else {
|
||||
updateData({
|
||||
nodes: [...data.nodes, ...newData.nodes],
|
||||
links: [...data.links, ...newData.links],
|
||||
});
|
||||
const onDataChange = useCallback((newData: NodesAndEdges) => {
|
||||
if (!newData.nodes.length && !newData.links.length) {
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
updateData({
|
||||
nodes: newData.nodes,
|
||||
links: newData.links,
|
||||
});
|
||||
}, []);
|
||||
|
||||
const graphRef = useRef<ForceGraphMethods>();
|
||||
|
||||
const graphControls = useRef<GraphControlsAPI>(null);
|
||||
|
||||
const onActivityChange = (activities: any) => {
|
||||
graphControls.current?.updateActivity(activities);
|
||||
};
|
||||
|
||||
const handleNodeClick = (node: NodeObject) => {
|
||||
graphControls.current?.setSelectedNode(node);
|
||||
graphRef.current?.d3ReheatSimulation();
|
||||
|
|
@ -63,29 +64,31 @@ export default function GraphView() {
|
|||
const addNodeDistanceFromSourceNode = 15;
|
||||
|
||||
const handleBackgroundClick = (event: MouseEvent) => {
|
||||
const graphBoundingBox = document.getElementById("graph-container")?.querySelector("canvas")?.getBoundingClientRect();
|
||||
const x = event.clientX - graphBoundingBox!.x;
|
||||
const y = event.clientY - graphBoundingBox!.y;
|
||||
|
||||
const graphClickCoords = graphRef.current!.screen2GraphCoords(x, y);
|
||||
|
||||
const selectedNode = graphControls.current?.getSelectedNode();
|
||||
|
||||
if (!selectedNode) {
|
||||
return;
|
||||
}
|
||||
|
||||
const distanceFromAddNode = Math.sqrt(
|
||||
Math.pow(graphClickCoords.x - (selectedNode!.x! + addNodeDistanceFromSourceNode), 2)
|
||||
+ Math.pow(graphClickCoords.y - (selectedNode!.y! + addNodeDistanceFromSourceNode), 2)
|
||||
);
|
||||
graphControls.current?.setSelectedNode(null);
|
||||
|
||||
if (distanceFromAddNode <= 10) {
|
||||
enableAddNodeForm();
|
||||
} else {
|
||||
disableAddNodeForm();
|
||||
graphControls.current?.setSelectedNode(null);
|
||||
}
|
||||
// const graphBoundingBox = document.getElementById("graph-container")?.querySelector("canvas")?.getBoundingClientRect();
|
||||
// const x = event.clientX - graphBoundingBox!.x;
|
||||
// const y = event.clientY - graphBoundingBox!.y;
|
||||
|
||||
// const graphClickCoords = graphRef.current!.screen2GraphCoords(x, y);
|
||||
|
||||
// const distanceFromAddNode = Math.sqrt(
|
||||
// Math.pow(graphClickCoords.x - (selectedNode!.x! + addNodeDistanceFromSourceNode), 2)
|
||||
// + Math.pow(graphClickCoords.y - (selectedNode!.y! + addNodeDistanceFromSourceNode), 2)
|
||||
// );
|
||||
|
||||
// if (distanceFromAddNode <= 10) {
|
||||
// enableAddNodeForm();
|
||||
// } else {
|
||||
// disableAddNodeForm();
|
||||
// graphControls.current?.setSelectedNode(null);
|
||||
// }
|
||||
};
|
||||
|
||||
function renderNode(node: NodeObject, ctx: CanvasRenderingContext2D, globalScale: number) {
|
||||
|
|
@ -93,23 +96,23 @@ export default function GraphView() {
|
|||
|
||||
ctx.save();
|
||||
|
||||
if (node.id === selectedNode?.id) {
|
||||
ctx.fillStyle = "gray";
|
||||
// if (node.id === selectedNode?.id) {
|
||||
// ctx.fillStyle = "gray";
|
||||
|
||||
ctx.beginPath();
|
||||
ctx.arc(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode, 10, 0, 2 * Math.PI);
|
||||
ctx.fill();
|
||||
// ctx.beginPath();
|
||||
// ctx.arc(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode, 10, 0, 2 * Math.PI);
|
||||
// ctx.fill();
|
||||
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(node.x! + addNodeDistanceFromSourceNode - 5, node.y! + addNodeDistanceFromSourceNode)
|
||||
ctx.lineTo(node.x! + addNodeDistanceFromSourceNode - 5 + 10, node.y! + addNodeDistanceFromSourceNode);
|
||||
ctx.stroke();
|
||||
// ctx.beginPath();
|
||||
// ctx.moveTo(node.x! + addNodeDistanceFromSourceNode - 5, node.y! + addNodeDistanceFromSourceNode)
|
||||
// ctx.lineTo(node.x! + addNodeDistanceFromSourceNode - 5 + 10, node.y! + addNodeDistanceFromSourceNode);
|
||||
// ctx.stroke();
|
||||
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode - 5)
|
||||
ctx.lineTo(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode - 5 + 10);
|
||||
ctx.stroke();
|
||||
}
|
||||
// ctx.beginPath();
|
||||
// ctx.moveTo(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode - 5)
|
||||
// ctx.lineTo(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode - 5 + 10);
|
||||
// ctx.stroke();
|
||||
// }
|
||||
|
||||
// ctx.beginPath();
|
||||
// ctx.arc(node.x, node.y, nodeSize, 0, 2 * Math.PI);
|
||||
|
|
@ -179,9 +182,7 @@ export default function GraphView() {
|
|||
ctx.restore();
|
||||
}
|
||||
|
||||
function handleDagError(loopNodeIds: (string | number)[]) {
|
||||
console.log(loopNodeIds);
|
||||
}
|
||||
function handleDagError(loopNodeIds: (string | number)[]) {}
|
||||
|
||||
useEffect(() => {
|
||||
// add collision force
|
||||
|
|
@ -211,7 +212,7 @@ export default function GraphView() {
|
|||
nodeRelSize={nodeSize}
|
||||
nodeCanvasObject={renderNode}
|
||||
nodeCanvasObjectMode={() => "after"}
|
||||
nodeAutoColorBy="group"
|
||||
nodeAutoColorBy="type"
|
||||
|
||||
linkLabel="label"
|
||||
linkCanvasObject={renderLink}
|
||||
|
|
@ -237,7 +238,7 @@ export default function GraphView() {
|
|||
nodeRelSize={20}
|
||||
nodeCanvasObject={renderNode}
|
||||
nodeCanvasObjectMode={() => "after"}
|
||||
nodeAutoColorBy="group"
|
||||
nodeAutoColorBy="type"
|
||||
|
||||
linkLabel="label"
|
||||
linkCanvasObject={renderLink}
|
||||
|
|
@ -250,10 +251,10 @@ export default function GraphView() {
|
|||
|
||||
<div className="absolute top-2 left-2 bg-gray-500 pt-4 pr-4 pb-4 pl-4 rounded-md max-w-2xl">
|
||||
<CogneeAddWidget onData={onDataChange} />
|
||||
<CrewAITrigger />
|
||||
<CrewAITrigger onData={onDataChange} onActivity={onActivityChange} />
|
||||
</div>
|
||||
|
||||
<div className="absolute top-2 right-2 bg-gray-500 pt-4 pr-4 pb-4 pl-4 rounded-md">
|
||||
<div className="absolute top-2 right-2 bg-gray-500 pt-4 pr-4 pb-4 pl-4 rounded-md w-110">
|
||||
<GraphControls
|
||||
ref={graphControls}
|
||||
isAddNodeFormOpen={isAddNodeFormOpen}
|
||||
|
|
@ -265,10 +266,12 @@ export default function GraphView() {
|
|||
<Divider />
|
||||
<div className="pl-6 pr-6">
|
||||
<Footer>
|
||||
<div className="flex flex-row items-center gap-6">
|
||||
<span>Nodes: {data?.nodes.length}</span>
|
||||
<span>Edges: {data?.links.length}</span>
|
||||
</div>
|
||||
{(data?.nodes.length || data?.links.length) && (
|
||||
<div className="flex flex-row items-center gap-6">
|
||||
<span>Nodes: {data?.nodes.length || 0}</span>
|
||||
<span>Edges: {data?.links.length || 0}</span>
|
||||
</div>
|
||||
)}
|
||||
</Footer>
|
||||
</div>
|
||||
</main>
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ function useDatasets() {
|
|||
|
||||
statusTimeout.current = setTimeout(() => {
|
||||
checkDatasetStatuses(datasets);
|
||||
}, 5000);
|
||||
}, 50000);
|
||||
}, [fetchDatasetStatuses]);
|
||||
|
||||
useEffect(() => {
|
||||
|
|
|
|||
|
|
@ -26,19 +26,22 @@ export default function FeedbackForm({ onSuccess }: FeedbackFormProps) {
|
|||
event.preventDefault();
|
||||
const formElements = event.currentTarget;
|
||||
|
||||
const authCredentials = new FormData();
|
||||
authCredentials.append("feedback", formElements.feedback.value);
|
||||
|
||||
setFeedbackError(null);
|
||||
disableFeedbackSubmit();
|
||||
|
||||
fetch("/v1/feedback/reasoning", {
|
||||
fetch("/v1/crewai/feedback", {
|
||||
method: "POST",
|
||||
body: authCredentials,
|
||||
body: JSON.stringify({
|
||||
feedback: formElements.feedback.value,
|
||||
}),
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(() => {
|
||||
onSuccess();
|
||||
formElements.feedback.value = "";
|
||||
})
|
||||
.catch(error => setFeedbackError(error.detail))
|
||||
.finally(() => enableFeedbackSubmit());
|
||||
|
|
@ -48,7 +51,7 @@ export default function FeedbackForm({ onSuccess }: FeedbackFormProps) {
|
|||
<form onSubmit={signIn} className="flex flex-col gap-2">
|
||||
<div className="flex flex-col gap-2">
|
||||
<div className="mb-4">
|
||||
<label className="block text-white" htmlFor="feedback">Your feedback on agents reasoning</label>
|
||||
<label className="block text-white" htmlFor="feedback">Feedback on agent's reasoning</label>
|
||||
<TextArea id="feedback" name="feedback" type="text" placeholder="Your feedback" />
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -47,9 +47,10 @@ app_environment = os.getenv("ENV", "prod")
|
|||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# from cognee.modules.data.deletion import prune_system, prune_data
|
||||
# await prune_data()
|
||||
# await prune_system(metadata = True)
|
||||
from cognee.modules.data.deletion import prune_system, prune_data
|
||||
|
||||
await prune_data()
|
||||
await prune_system(metadata=True)
|
||||
# if app_environment == "local" or app_environment == "dev":
|
||||
from cognee.infrastructure.databases.relational import get_relational_engine
|
||||
|
||||
|
|
@ -189,7 +190,7 @@ def start_api_server(host: str = "0.0.0.0", port: int = 8000):
|
|||
try:
|
||||
logger.info("Starting server at %s:%s", host, port)
|
||||
|
||||
uvicorn.run(app, host=host, port=port)
|
||||
uvicorn.run(app, host=host, port=port, loop="asyncio")
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to start server: {e}")
|
||||
# Here you could add any cleanup code or error recovery code.
|
||||
|
|
|
|||
|
|
@ -2,17 +2,19 @@ import asyncio
|
|||
from pydantic import BaseModel
|
||||
from typing import Union, Optional
|
||||
|
||||
from cognee.modules.users.methods import get_default_user
|
||||
from cognee.shared.logging_utils import get_logger
|
||||
from cognee.shared.data_models import KnowledgeGraph
|
||||
from cognee.infrastructure.llm import get_max_chunk_tokens
|
||||
|
||||
from cognee.modules.users.models import User
|
||||
from cognee.modules.pipelines import cognee_pipeline
|
||||
from cognee.modules.pipelines.tasks.task import Task
|
||||
from cognee.modules.chunking.TextChunker import TextChunker
|
||||
from cognee.modules.ontology.rdf_xml.OntologyResolver import OntologyResolver
|
||||
from cognee.modules.pipelines.models.PipelineRunInfo import PipelineRunCompleted
|
||||
from cognee.modules.pipelines.models.PipelineRunInfo import PipelineRunCompleted, PipelineRunStarted
|
||||
from cognee.modules.pipelines.queues.pipeline_run_info_queues import push_to_queue
|
||||
from cognee.modules.users.models import User
|
||||
from cognee.modules.graph.operations import get_formatted_graph_data
|
||||
from cognee.modules.crewai.get_crewai_pipeline_run_id import get_crewai_pipeline_run_id
|
||||
|
||||
from cognee.tasks.documents import (
|
||||
check_permissions_on_documents,
|
||||
|
|
@ -36,16 +38,20 @@ async def cognify(
|
|||
chunk_size: int = None,
|
||||
ontology_file_path: Optional[str] = None,
|
||||
run_in_background: bool = False,
|
||||
is_stream_info_enabled: bool = False,
|
||||
):
|
||||
tasks = await get_default_tasks(user, graph_model, chunker, chunk_size, ontology_file_path)
|
||||
|
||||
if not user:
|
||||
user = await get_default_user()
|
||||
|
||||
if run_in_background:
|
||||
return await run_cognify_as_background_process(tasks, user, datasets)
|
||||
else:
|
||||
return await run_cognify_blocking(tasks, user, datasets)
|
||||
return await run_cognify_blocking(tasks, user, datasets, is_stream_info_enabled)
|
||||
|
||||
|
||||
async def run_cognify_blocking(tasks, user, datasets):
|
||||
async def run_cognify_blocking(tasks, user, datasets, is_stream_info_enabled=False):
|
||||
pipeline_run_info = None
|
||||
|
||||
async for run_info in cognee_pipeline(
|
||||
|
|
@ -53,6 +59,15 @@ async def run_cognify_blocking(tasks, user, datasets):
|
|||
):
|
||||
pipeline_run_info = run_info
|
||||
|
||||
if (
|
||||
is_stream_info_enabled
|
||||
and not isinstance(pipeline_run_info, PipelineRunStarted)
|
||||
and not isinstance(pipeline_run_info, PipelineRunCompleted)
|
||||
):
|
||||
pipeline_run_id = get_crewai_pipeline_run_id(user.id)
|
||||
pipeline_run_info.payload = await get_formatted_graph_data()
|
||||
push_to_queue(pipeline_run_id, pipeline_run_info)
|
||||
|
||||
return pipeline_run_info
|
||||
|
||||
|
||||
|
|
@ -68,6 +83,8 @@ async def run_cognify_as_background_process(tasks, user, datasets):
|
|||
try:
|
||||
pipeline_run_info = await anext(pipeline_run)
|
||||
|
||||
pipeline_run_info.payload = await get_formatted_graph_data()
|
||||
|
||||
push_to_queue(pipeline_run_info.pipeline_run_id, pipeline_run_info)
|
||||
|
||||
if isinstance(pipeline_run_info, PipelineRunCompleted):
|
||||
|
|
|
|||
|
|
@ -69,11 +69,15 @@ def get_cognify_router() -> APIRouter:
|
|||
continue
|
||||
|
||||
try:
|
||||
await websocket.send_json({
|
||||
"pipeline_run_id": str(pipeline_run_info.pipeline_run_id),
|
||||
"status": pipeline_run_info.status,
|
||||
"payload": await get_nodes_and_edges(pipeline_run_info.payload) if pipeline_run_info.payload else None,
|
||||
})
|
||||
await websocket.send_json(
|
||||
{
|
||||
"pipeline_run_id": str(pipeline_run_info.pipeline_run_id),
|
||||
"status": pipeline_run_info.status,
|
||||
"payload": await get_nodes_and_edges(pipeline_run_info.payload)
|
||||
if pipeline_run_info.payload
|
||||
else None,
|
||||
}
|
||||
)
|
||||
|
||||
if isinstance(pipeline_run_info, PipelineRunCompleted):
|
||||
remove_queue(pipeline_run_id)
|
||||
|
|
@ -85,6 +89,7 @@ def get_cognify_router() -> APIRouter:
|
|||
|
||||
return router
|
||||
|
||||
|
||||
async def get_nodes_and_edges(data_points):
|
||||
nodes = []
|
||||
edges = []
|
||||
|
|
@ -112,14 +117,24 @@ async def get_nodes_and_edges(data_points):
|
|||
nodes, edges = deduplicate_nodes_and_edges(nodes, edges)
|
||||
|
||||
return {
|
||||
"nodes": list(map(lambda node: {
|
||||
"id": str(node.id),
|
||||
"label": node.name if hasattr(node, "name") else f"{node.type}_{str(node.id)}",
|
||||
"properties": {},
|
||||
}, nodes)),
|
||||
"edges": list(map(lambda edge: {
|
||||
"source": str(edge[0]),
|
||||
"target": str(edge[1]),
|
||||
"label": edge[2],
|
||||
}, edges)),
|
||||
"nodes": list(
|
||||
map(
|
||||
lambda node: {
|
||||
"id": str(node.id),
|
||||
"label": node.name if hasattr(node, "name") else f"{node.type}_{str(node.id)}",
|
||||
"properties": {},
|
||||
},
|
||||
nodes,
|
||||
)
|
||||
),
|
||||
"edges": list(
|
||||
map(
|
||||
lambda edge: {
|
||||
"source": str(edge[0]),
|
||||
"target": str(edge[1]),
|
||||
"label": edge[2],
|
||||
},
|
||||
edges,
|
||||
)
|
||||
),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,23 +1,117 @@
|
|||
from fastapi import APIRouter, Depends
|
||||
import os
|
||||
import asyncio
|
||||
from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect
|
||||
from starlette.status import WS_1000_NORMAL_CLOSURE, WS_1008_POLICY_VIOLATION
|
||||
|
||||
from cognee.api.DTO import InDTO
|
||||
from cognee.modules.users.get_fastapi_users import get_fastapi_users
|
||||
from cognee.modules.users.authentication.get_auth_backend import get_auth_backend
|
||||
from cognee.modules.users.methods import get_authenticated_user
|
||||
from cognee.complex_demos.crewai_demo.src.crewai_demo.github_ingest_datapoints import (
|
||||
cognify_github_data_from_username,
|
||||
)
|
||||
from cognee.modules.crewai.get_crewai_pipeline_run_id import get_crewai_pipeline_run_id
|
||||
from cognee.modules.users.models import User
|
||||
from cognee.modules.users.methods import get_authenticated_user
|
||||
from cognee.modules.pipelines.models import PipelineRunInfo, PipelineRunCompleted
|
||||
from cognee.complex_demos.crewai_demo.src.crewai_demo.main import (
|
||||
# run_github_ingestion,
|
||||
run_hiring_crew,
|
||||
)
|
||||
from cognee.modules.pipelines.queues.pipeline_run_info_queues import (
|
||||
get_from_queue,
|
||||
initialize_queue,
|
||||
remove_queue,
|
||||
)
|
||||
|
||||
|
||||
class CrewAIRunPayloadDTO(InDTO):
|
||||
username1: str
|
||||
username2: str
|
||||
|
||||
|
||||
class CrewAIFeedbackPayloadDTO(InDTO):
|
||||
feedback: str
|
||||
|
||||
|
||||
def get_crewai_router() -> APIRouter:
|
||||
router = APIRouter()
|
||||
|
||||
@router.post("/run", response_model=str)
|
||||
async def run_crewai(payload: CrewAIRunPayloadDTO, user: User = Depends(get_authenticated_user)):
|
||||
@router.post("/run", response_model=bool)
|
||||
async def run_crewai(
|
||||
payload: CrewAIRunPayloadDTO,
|
||||
user: User = Depends(get_authenticated_user),
|
||||
):
|
||||
# Run CrewAI with the provided usernames
|
||||
print(payload.username1, payload.username2)
|
||||
# run_future = run_github_ingestion(payload.username1, payload.username2)
|
||||
token = os.getenv("GITHUB_TOKEN")
|
||||
|
||||
return "CrewAI run started"
|
||||
await cognify_github_data_from_username(payload.username1, token)
|
||||
await cognify_github_data_from_username(payload.username2, token)
|
||||
|
||||
applicants = {
|
||||
"applicant_1": payload.username1,
|
||||
"applicant_2": payload.username2,
|
||||
}
|
||||
|
||||
run_hiring_crew(applicants=applicants, number_of_rounds=2)
|
||||
|
||||
return True
|
||||
|
||||
@router.post("/feedback", response_model=None)
|
||||
async def send_feedback(
|
||||
payload: CrewAIFeedbackPayloadDTO,
|
||||
user: User = Depends(
|
||||
get_authenticated_user,
|
||||
),
|
||||
):
|
||||
from cognee import add, cognify
|
||||
# from secrets import choice
|
||||
# from string import ascii_letters, digits
|
||||
|
||||
# hash6 = "".join(choice(ascii_letters + digits) for _ in range(6))
|
||||
dataset_name = "final_reports"
|
||||
await add(payload.feedback, node_set=["final_report"], dataset_name=dataset_name)
|
||||
await cognify(datasets=dataset_name, is_stream_info_enabled=True)
|
||||
|
||||
@router.websocket("/subscribe")
|
||||
async def subscribe_to_crewai_info(websocket: WebSocket):
|
||||
await websocket.accept()
|
||||
|
||||
auth_message = await websocket.receive_json()
|
||||
|
||||
try:
|
||||
user = await get_authenticated_user(auth_message.get("Authorization"))
|
||||
except Exception:
|
||||
await websocket.close(code=WS_1008_POLICY_VIOLATION, reason="Unauthorized")
|
||||
return
|
||||
|
||||
pipeline_run_id = get_crewai_pipeline_run_id(user.id)
|
||||
|
||||
initialize_queue(pipeline_run_id)
|
||||
|
||||
while True:
|
||||
pipeline_run_info = get_from_queue(pipeline_run_id)
|
||||
|
||||
if not pipeline_run_info:
|
||||
await asyncio.sleep(2)
|
||||
continue
|
||||
|
||||
if not isinstance(pipeline_run_info, PipelineRunInfo):
|
||||
continue
|
||||
|
||||
try:
|
||||
await websocket.send_json(
|
||||
{
|
||||
"pipeline_run_id": str(pipeline_run_info.pipeline_run_id),
|
||||
"status": pipeline_run_info.status,
|
||||
"payload": pipeline_run_info.payload if pipeline_run_info.payload else None,
|
||||
}
|
||||
)
|
||||
|
||||
if isinstance(pipeline_run_info, PipelineRunCompleted):
|
||||
remove_queue(pipeline_run_id)
|
||||
await websocket.close(code=WS_1000_NORMAL_CLOSURE)
|
||||
break
|
||||
except WebSocketDisconnect:
|
||||
remove_queue(pipeline_run_id)
|
||||
break
|
||||
|
||||
return router
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from cognee.modules.graph.operations import get_formatted_graph_data
|
||||
from cognee.shared.logging_utils import get_logger
|
||||
from fastapi import APIRouter
|
||||
from datetime import datetime
|
||||
|
|
@ -44,11 +45,13 @@ class GraphNodeDTO(OutDTO):
|
|||
label: str
|
||||
properties: dict
|
||||
|
||||
|
||||
class GraphEdgeDTO(OutDTO):
|
||||
source: UUID
|
||||
target: UUID
|
||||
label: str
|
||||
|
||||
|
||||
class GraphDTO(OutDTO):
|
||||
nodes: List[GraphNodeDTO]
|
||||
edges: List[GraphEdgeDTO]
|
||||
|
|
@ -111,26 +114,10 @@ def get_datasets_router() -> APIRouter:
|
|||
|
||||
@router.get("/{dataset_id}/graph", response_model=GraphDTO)
|
||||
async def get_dataset_graph(dataset_id: UUID, user: User = Depends(get_authenticated_user)):
|
||||
from cognee.infrastructure.databases.graph import get_graph_engine
|
||||
|
||||
try:
|
||||
graph_client = await get_graph_engine()
|
||||
(nodes, edges) = await graph_client.get_graph_data()
|
||||
|
||||
return JSONResponse(
|
||||
status_code=200,
|
||||
content={
|
||||
"nodes": list(map(lambda node: {
|
||||
"id": str(node[0]),
|
||||
"label": node[1]["name"] if hasattr(node[1], "name") else f"{node[1]['type']}_{str(node[0])}",
|
||||
"properties": {},
|
||||
}, nodes)),
|
||||
"edges": list(map(lambda edge: {
|
||||
"source": str(edge[0]),
|
||||
"target": str(edge[1]),
|
||||
"label": edge[2],
|
||||
}, edges)),
|
||||
},
|
||||
content=await get_formatted_graph_data(),
|
||||
)
|
||||
except Exception as error:
|
||||
print(error)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ class CogneeBuild(BaseTool):
|
|||
node_set = meta["nodeset"]
|
||||
await cognee.add(text, node_set=node_set)
|
||||
|
||||
await cognee.cognify()
|
||||
await cognee.cognify(is_stream_info_enabled=True)
|
||||
|
||||
return "Knowledge Graph is done."
|
||||
except Exception as e:
|
||||
|
|
@ -27,10 +27,11 @@ class CogneeBuild(BaseTool):
|
|||
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
if loop.is_running():
|
||||
|
||||
if not loop.is_running():
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
results = loop.run_until_complete(main())
|
||||
return results
|
||||
|
||||
return loop.create_task(main())
|
||||
except Exception as e:
|
||||
return f"Tool execution error: {str(e)}"
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
from crewai.tools import BaseTool
|
||||
from typing import Type, List
|
||||
from pydantic import BaseModel, Field, PrivateAttr
|
||||
from cognee.modules.engine.models import NodeSet
|
||||
import asyncio
|
||||
import nest_asyncio
|
||||
from crewai.tools import BaseTool
|
||||
from typing import Type
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class CogneeIngestionInput(BaseModel):
|
||||
|
|
@ -24,14 +24,15 @@ class CogneeIngestion(BaseTool):
|
|||
|
||||
def _run(self, text: str) -> str:
|
||||
import cognee
|
||||
from secrets import choice
|
||||
from string import ascii_letters, digits
|
||||
# from secrets import choice
|
||||
# from string import ascii_letters, digits
|
||||
|
||||
async def main():
|
||||
try:
|
||||
hash6 = "".join(choice(ascii_letters + digits) for _ in range(6))
|
||||
await cognee.add(text, node_set=[self._nodeset_name], dataset_name=hash6)
|
||||
await cognee.cognify(datasets=hash6)
|
||||
# hash6 = "".join(choice(ascii_letters + digits) for _ in range(6))
|
||||
dataset_name = "final_reports"
|
||||
await cognee.add(text, node_set=[self._nodeset_name], dataset_name=dataset_name)
|
||||
await cognee.cognify(datasets=dataset_name, is_stream_info_enabled=True)
|
||||
|
||||
return "Report ingested successfully into Cognee memory."
|
||||
except Exception as e:
|
||||
|
|
@ -39,9 +40,15 @@ class CogneeIngestion(BaseTool):
|
|||
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
if loop.is_running():
|
||||
|
||||
if not loop.is_running():
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
return loop.run_until_complete(main())
|
||||
|
||||
nest_asyncio.apply(loop)
|
||||
|
||||
result = loop.run_until_complete(main())
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
return f"Tool execution error: {str(e)}"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import nest_asyncio
|
||||
|
||||
from crewai.tools import BaseTool
|
||||
from typing import Type
|
||||
from pydantic import BaseModel, Field, PrivateAttr
|
||||
|
|
@ -47,10 +49,15 @@ class CogneeSearch(BaseTool):
|
|||
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
if loop.is_running():
|
||||
|
||||
if not loop.is_running():
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
search_results = loop.run_until_complete(main())
|
||||
return search_results
|
||||
|
||||
nest_asyncio.apply(loop)
|
||||
|
||||
result = loop.run_until_complete(main())
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
return f"Tool execution error: {str(e)}"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
from crewai.tools import BaseTool
|
||||
|
||||
from cognee.modules.engine.models import NodeSet
|
||||
from cognee.modules.retrieval.graph_completion_retriever import GraphCompletionRetriever
|
||||
from ..github_ingest_datapoints import cognify_github_data_from_username
|
||||
|
||||
|
||||
|
|
@ -11,30 +9,32 @@ class GithubIngestion(BaseTool):
|
|||
|
||||
def _run(self, applicant_1, applicant_2) -> str:
|
||||
import asyncio
|
||||
import cognee
|
||||
|
||||
# import cognee
|
||||
import os
|
||||
from cognee.low_level import DataPoint, setup as cognee_setup
|
||||
# from cognee.low_level import setup as cognee_setup
|
||||
|
||||
async def main():
|
||||
try:
|
||||
await cognee.prune.prune_data()
|
||||
await cognee.prune.prune_system(metadata=True)
|
||||
await cognee_setup()
|
||||
# await cognee.prune.prune_data()
|
||||
# await cognee.prune.prune_system(metadata=True)
|
||||
# await cognee_setup()
|
||||
token = os.getenv("GITHUB_TOKEN")
|
||||
|
||||
await cognify_github_data_from_username(applicant_1, token)
|
||||
await cognify_github_data_from_username(applicant_2, token)
|
||||
|
||||
return "Github ingestion finished"
|
||||
return True
|
||||
except Exception as e:
|
||||
return f"Error: {str(e)}"
|
||||
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
if loop.is_running():
|
||||
|
||||
if not loop.is_running():
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
results = loop.run_until_complete(main())
|
||||
return results
|
||||
|
||||
return loop.create_task(main())
|
||||
except Exception as e:
|
||||
return f"Tool execution error: {str(e)}"
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from uuid import uuid5, NAMESPACE_OID
|
||||
from typing import Dict, Any, List, Tuple, Optional
|
||||
from typing import Dict, Any, List
|
||||
|
||||
from cognee.low_level import DataPoint
|
||||
from cognee.modules.engine.models.node_set import NodeSet
|
||||
from cognee.shared.logging_utils import get_logger
|
||||
from cognee.complex_demos.crewai_demo.src.crewai_demo.github_datapoints import (
|
||||
|
|
@ -60,7 +59,9 @@ def create_file_datapoint(filename: str, repo_name: str, nodesets: List[NodeSet]
|
|||
"""Creates a File DataPoint with a consistent ID."""
|
||||
file_key = f"{repo_name}:{filename}"
|
||||
file_id = uuid5(NAMESPACE_OID, file_key)
|
||||
file = File(id=file_id, filename=filename, repo=repo_name, belongs_to_set=nodesets)
|
||||
file = File(
|
||||
id=file_id, name=filename, filename=filename, repo=repo_name, belongs_to_set=nodesets
|
||||
)
|
||||
logger.debug(f"Created File with ID: {file_id} for {filename}")
|
||||
return file
|
||||
|
||||
|
|
@ -72,6 +73,7 @@ def create_commit_datapoint(
|
|||
commit_id = uuid5(NAMESPACE_OID, commit_data.get("commit_sha", ""))
|
||||
commit = Commit(
|
||||
id=commit_id,
|
||||
name=commit_data.get("commit_sha", ""),
|
||||
commit_sha=commit_data.get("commit_sha", ""),
|
||||
text="Commit message:" + (str)(commit_data.get("commit_message", "")),
|
||||
commit_date=commit_data.get("commit_date", ""),
|
||||
|
|
@ -96,6 +98,7 @@ def create_file_change_datapoint(
|
|||
|
||||
file_change = FileChange(
|
||||
id=fc_id,
|
||||
name=fc_data.get("filename", ""),
|
||||
filename=fc_data.get("filename", ""),
|
||||
status=fc_data.get("status", ""),
|
||||
additions=fc_data.get("additions", 0),
|
||||
|
|
@ -121,6 +124,7 @@ def create_issue_datapoint(
|
|||
|
||||
issue = Issue(
|
||||
id=issue_id,
|
||||
name=str(issue_data.get("issue_number", 0)),
|
||||
number=issue_data.get("issue_number", 0),
|
||||
text=issue_data.get("issue_title", ""),
|
||||
state=issue_data.get("issue_state", ""),
|
||||
|
|
@ -142,6 +146,7 @@ def create_comment_datapoint(
|
|||
|
||||
comment = Comment(
|
||||
id=comment_id,
|
||||
name=str(comment_data.get("comment_id", "")),
|
||||
comment_id=str(comment_data.get("comment_id", "")),
|
||||
text=comment_data.get("body", ""),
|
||||
created_at=comment_data.get("created_at", ""),
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ class File(DataPoint):
|
|||
"""File is now a leaf node without any lists of other DataPoints"""
|
||||
|
||||
filename: str
|
||||
name: str
|
||||
repo: str
|
||||
metadata: dict = {"index_fields": ["filename"]}
|
||||
|
||||
|
|
@ -25,6 +26,7 @@ class GitHubUser(DataPoint):
|
|||
|
||||
class FileChange(DataPoint):
|
||||
filename: str
|
||||
name: str
|
||||
status: str
|
||||
additions: int
|
||||
deletions: int
|
||||
|
|
@ -39,6 +41,7 @@ class FileChange(DataPoint):
|
|||
|
||||
class Comment(DataPoint):
|
||||
comment_id: str
|
||||
name: str
|
||||
text: str
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
|
@ -51,6 +54,7 @@ class Comment(DataPoint):
|
|||
|
||||
class Issue(DataPoint):
|
||||
number: int
|
||||
name: str
|
||||
text: str
|
||||
state: str
|
||||
repository: str
|
||||
|
|
@ -60,6 +64,7 @@ class Issue(DataPoint):
|
|||
|
||||
class Commit(DataPoint):
|
||||
commit_sha: str
|
||||
name: str
|
||||
text: str
|
||||
commit_date: str
|
||||
commit_url: str
|
||||
|
|
|
|||
|
|
@ -2,11 +2,8 @@ import json
|
|||
import asyncio
|
||||
from uuid import uuid5, NAMESPACE_OID
|
||||
from typing import Optional, List, Dict, Any
|
||||
from pathlib import Path
|
||||
from cognee.api.v1.search import SearchType
|
||||
import cognee
|
||||
from cognee.low_level import DataPoint, setup as cognee_setup
|
||||
from cognee.modules.retrieval.graph_completion_retriever import GraphCompletionRetriever
|
||||
from cognee.low_level import DataPoint
|
||||
from cognee.tasks.storage import add_data_points
|
||||
from cognee.modules.pipelines.tasks.task import Task
|
||||
from cognee.modules.pipelines import run_tasks
|
||||
|
|
@ -16,6 +13,9 @@ from cognee.shared.logging_utils import get_logger
|
|||
from cognee.complex_demos.crewai_demo.src.crewai_demo.github_ingest import (
|
||||
get_github_data_for_cognee,
|
||||
)
|
||||
from cognee.modules.pipelines.models.PipelineRunInfo import PipelineRunCompleted, PipelineRunStarted
|
||||
from cognee.modules.graph.operations import get_formatted_graph_data
|
||||
from cognee.modules.crewai.get_crewai_pipeline_run_id import get_crewai_pipeline_run_id
|
||||
|
||||
# Import DataPoint classes from github_datapoints.py
|
||||
from cognee.complex_demos.crewai_demo.src.crewai_demo.github_datapoints import (
|
||||
|
|
@ -205,6 +205,27 @@ def build_github_datapoints_from_dict(github_data: Dict[str, Any]):
|
|||
return all_datapoints
|
||||
|
||||
|
||||
async def run_with_info_stream(tasks, user, data, dataset_id, pipeline_name):
|
||||
from cognee.modules.pipelines.queues.pipeline_run_info_queues import push_to_queue
|
||||
|
||||
pipeline_run = run_tasks(
|
||||
tasks=tasks,
|
||||
data=data,
|
||||
dataset_id=dataset_id,
|
||||
pipeline_name=pipeline_name,
|
||||
user=user,
|
||||
)
|
||||
|
||||
pipeline_run_id = get_crewai_pipeline_run_id(user.id)
|
||||
|
||||
async for pipeline_run_info in pipeline_run:
|
||||
if not isinstance(pipeline_run_info, PipelineRunStarted) and not isinstance(
|
||||
pipeline_run_info, PipelineRunCompleted
|
||||
):
|
||||
pipeline_run_info.payload = await get_formatted_graph_data()
|
||||
push_to_queue(pipeline_run_id, pipeline_run_info)
|
||||
|
||||
|
||||
async def cognify_github_data(github_data: dict):
|
||||
"""Process GitHub user, file changes, and comments data from a loaded dictionary."""
|
||||
all_datapoints = build_github_datapoints_from_dict(github_data)
|
||||
|
|
@ -216,18 +237,16 @@ async def cognify_github_data(github_data: dict):
|
|||
|
||||
cognee_user = await get_default_user()
|
||||
tasks = [Task(add_data_points, task_config={"batch_size": 50})]
|
||||
results = run_tasks(
|
||||
|
||||
await run_with_info_stream(
|
||||
tasks=tasks,
|
||||
data=all_datapoints,
|
||||
dataset_id=dataset_id,
|
||||
pipeline_name="github_pipeline",
|
||||
user=cognee_user,
|
||||
)
|
||||
async for result in results:
|
||||
print(result)
|
||||
|
||||
logger.info(f"Done processing {len(all_datapoints)} datapoints")
|
||||
return True
|
||||
|
||||
|
||||
async def cognify_github_data_from_username(
|
||||
|
|
@ -263,8 +282,6 @@ async def cognify_github_data_from_username(
|
|||
|
||||
await cognify_github_data(github_data)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def process_github_from_file(json_file_path: str):
|
||||
"""Process GitHub data from a JSON file."""
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import os
|
||||
from crewai import Agent, Crew, Process, Task
|
||||
from crewai.project import CrewBase, agent, crew, task, before_kickoff
|
||||
from crewai.project import CrewBase, agent, crew, task
|
||||
from pydantic import BaseModel
|
||||
|
||||
from cognee.complex_demos.crewai_demo.src.crewai_demo.custom_tools.cognee_ingestion import (
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import warnings
|
||||
import os
|
||||
from hiring_crew import HiringCrew
|
||||
from .hiring_crew import HiringCrew
|
||||
from cognee.complex_demos.crewai_demo.src.crewai_demo.custom_tools.github_ingestion import (
|
||||
GithubIngestion,
|
||||
)
|
||||
|
|
@ -14,7 +14,7 @@ def print_environment():
|
|||
|
||||
|
||||
def run_github_ingestion(applicant_1, applicant_2):
|
||||
GithubIngestion().run(applicant_1=applicant_1, applicant_2=applicant_2)
|
||||
return GithubIngestion().run(applicant_1=applicant_1, applicant_2=applicant_2)
|
||||
|
||||
|
||||
def run_hiring_crew(applicants: dict, number_of_rounds: int = 1, llm_client=None):
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ class OpenAIAdapter(LLMInterface):
|
|||
self.max_tokens = max_tokens
|
||||
self.streaming = streaming
|
||||
|
||||
@observe(as_type="generation")
|
||||
# @observe(as_type="generation")
|
||||
@sleep_and_retry_async()
|
||||
@rate_limit_async
|
||||
async def acreate_structured_output(
|
||||
|
|
@ -77,7 +77,7 @@ class OpenAIAdapter(LLMInterface):
|
|||
max_retries=self.MAX_RETRIES,
|
||||
)
|
||||
|
||||
@observe
|
||||
# @observe
|
||||
@sleep_and_retry_sync()
|
||||
@rate_limit_sync
|
||||
def create_structured_output(
|
||||
|
|
|
|||
11
cognee/modules/crewai/get_crewai_pipeline_run_id.py
Normal file
11
cognee/modules/crewai/get_crewai_pipeline_run_id.py
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
from uuid import NAMESPACE_OID, UUID, uuid5
|
||||
|
||||
from cognee.modules.pipelines.utils import generate_pipeline_id, generate_pipeline_run_id
|
||||
|
||||
|
||||
def get_crewai_pipeline_run_id(user_id: UUID):
|
||||
dataset_id = uuid5(NAMESPACE_OID, "GitHub")
|
||||
pipeline_id = generate_pipeline_id(user_id, "github_pipeline")
|
||||
pipeline_run_id = generate_pipeline_run_id(pipeline_id, dataset_id)
|
||||
|
||||
return pipeline_run_id
|
||||
1
cognee/modules/graph/operations/__init__.py
Normal file
1
cognee/modules/graph/operations/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
from .get_formatted_graph_data import get_formatted_graph_data
|
||||
36
cognee/modules/graph/operations/get_formatted_graph_data.py
Normal file
36
cognee/modules/graph/operations/get_formatted_graph_data.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
from cognee.infrastructure.databases.graph import get_graph_engine
|
||||
|
||||
|
||||
async def get_formatted_graph_data():
|
||||
graph_client = await get_graph_engine()
|
||||
(nodes, edges) = await graph_client.get_graph_data()
|
||||
|
||||
return {
|
||||
"nodes": list(
|
||||
map(
|
||||
lambda node: {
|
||||
"id": str(node[0]),
|
||||
"label": node[1]["name"]
|
||||
if ("name" in node[1] and node[1]["name"] != "")
|
||||
else f"{node[1]['type']}_{str(node[0])}",
|
||||
"type": node[1]["type"],
|
||||
"properties": {
|
||||
key: value
|
||||
for key, value in node[1].items()
|
||||
if key not in ["id", "type", "name"] and value is not None
|
||||
},
|
||||
},
|
||||
nodes,
|
||||
)
|
||||
),
|
||||
"edges": list(
|
||||
map(
|
||||
lambda edge: {
|
||||
"source": str(edge[0]),
|
||||
"target": str(edge[1]),
|
||||
"label": edge[2],
|
||||
},
|
||||
edges,
|
||||
)
|
||||
),
|
||||
}
|
||||
|
|
@ -15,19 +15,19 @@ class PipelineRunInfo(BaseModel):
|
|||
|
||||
class PipelineRunStarted(PipelineRunInfo):
|
||||
status: str = "PipelineRunStarted"
|
||||
pass
|
||||
|
||||
|
||||
class PipelineRunYield(PipelineRunInfo):
|
||||
status: str = "PipelineRunYield"
|
||||
pass
|
||||
|
||||
|
||||
class PipelineRunCompleted(PipelineRunInfo):
|
||||
status: str = "PipelineRunCompleted"
|
||||
pass
|
||||
|
||||
|
||||
class PipelineRunErrored(PipelineRunInfo):
|
||||
status: str = "PipelineRunErrored"
|
||||
pass
|
||||
|
||||
|
||||
class PipelineRunActivity(BaseModel):
|
||||
status: str = "PipelineRunActivity"
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ authors = [
|
|||
{ name = "Vasilije Markovic" },
|
||||
{ name = "Boris Arzentar" },
|
||||
]
|
||||
requires-python = ">=3.10,<=3.13"
|
||||
requires-python = ">=3.10,<3.13"
|
||||
readme = "README.md"
|
||||
license = "Apache-2.0"
|
||||
classifiers = [
|
||||
|
|
@ -58,6 +58,7 @@ dependencies = [
|
|||
"structlog>=25.2.0,<26",
|
||||
"onnxruntime<=1.21.1",
|
||||
"pylance==0.22.0",
|
||||
"nest-asyncio>=1.6.0",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
|
@ -91,10 +92,6 @@ falkordb = ["falkordb==1.0.9"]
|
|||
kuzu = ["kuzu==0.8.2"]
|
||||
groq = ["groq==0.8.0"]
|
||||
milvus = ["pymilvus>=2.5.0,<3"]
|
||||
chromadb = [
|
||||
"chromadb>=0.3.0,<0.7",
|
||||
"pypika==0.48.8",
|
||||
]
|
||||
docs = ["unstructured[csv, doc, docx, epub, md, odt, org, ppt, pptx, rst, rtf, tsv, xlsx]>=0.16.13,<0.17"]
|
||||
codegraph = [
|
||||
"fastembed<=0.6.0 ; python_version < '3.13'",
|
||||
|
|
@ -128,6 +125,10 @@ dev = [
|
|||
"mkdocstrings[python]>=0.26.2,<0.27",
|
||||
]
|
||||
debug = ["debugpy==1.8.9"]
|
||||
crewai = [
|
||||
"crewai>=0.117.1",
|
||||
"pygithub>=2.6.1",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://www.cognee.ai"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue