gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: websockets for pipeline update streaming (#851)

Browse source 
<!-- .github/pull_request_template.md -->

## Description
<!-- Provide a clear description of the changes in this PR -->

## DCO Affirmation
I affirm that all code in every commit of this pull request conforms to
the terms of the Topoteretes Developer Certificate of Origin.

---------

Co-authored-by: hajdul88 <52442977+hajdul88@users.noreply.github.com>
Co-authored-by: lxobr <122801072+lxobr@users.noreply.github.com>
Co-authored-by: Igor Ilic <30923996+dexters1@users.noreply.github.com>
Co-authored-by: Hande <159312713+hande-k@users.noreply.github.com>
Co-authored-by: Vasilije <8619304+Vasilije1990@users.noreply.github.com>
This commit is contained in:
Boris 2025-06-11 20:29:26 +02:00 • committed by GitHub
parent d2e45a2118
commit 773b15a645
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
63 changed files with 3736 additions and 661 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cognee-frontend/.prettierrc Normal file
View file

@ -0,0 +1,7 @@
{
"trailingComma": "es5",
"tabWidth": 2,
"semi": true,
"singleQuote": false,
"plugins": ["prettier-plugin-tailwindcss"]
}

1138
cognee-frontend/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

10
cognee-frontend/package.json
View file

@ -1,6 +1,6 @@
{
"name": "cognee-frontend",
"version": "0.1.0",
"version": "1.0.0",
"private": true,
"scripts": {
"dev": "next dev",
@ -10,13 +10,17 @@
},
"dependencies": {
"classnames": "^2.5.1",
"d3-force-3d": "^3.0.6",
"next": "15.3.2",
"ohmy-ui": "^0.0.6",
"react": "^18",
"react-dom": "^18",
"uuid": "^9.0.1",
"next": "^14.2.26"
"react-force-graph-2d": "^1.27.1",
"tailwindcss": "^4.1.7",
"uuid": "^9.0.1"
},
"devDependencies": {
"@tailwindcss/postcss": "^4.1.7",
"@types/node": "^20",
"@types/react": "^18",
"@types/react-dom": "^18",

5
cognee-frontend/postcss.config.mjs Normal file
View file

@ -0,0 +1,5 @@
export default {
plugins: {
"@tailwindcss/postcss": {},
}
}

89
cognee-frontend/src/app/(graph)/CogneeAddWidget.tsx Normal file
View file

@ -0,0 +1,89 @@
"use client";
import { ChangeEvent, useEffect } from "react";
import { CTAButton, StatusIndicator } from "@/ui/elements";
import addData from "@/modules/ingestion/addData";
import cognifyDataset from "@/modules/datasets/cognifyDataset";
import useDatasets from "@/modules/ingestion/useDatasets";
import getDatasetGraph from '@/modules/datasets/getDatasetGraph';
export interface NodesAndEdges {
nodes: { id: string; label: string }[];
links: { source: string; target: string; label: string }[];
}
interface CogneeAddWidgetProps {
onData: (data: NodesAndEdges) => void;
}
export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
const {
datasets,
addDataset,
removeDataset,
refreshDatasets,
} = useDatasets();
useEffect(() => {
refreshDatasets()
.then((datasets) => {
const dataset = datasets?.[0];
if (dataset) {
getDatasetGraph(dataset)
.then((graph) => onData({
nodes: graph.nodes,
links: graph.edges,
}));
}
});
}, [refreshDatasets]);
const handleAddFiles = (dataset: { id?: string, name?: string }, event: ChangeEvent<HTMLInputElement>) => {
event.stopPropagation();
if (!event.currentTarget.files) {
throw new Error("Error: No files added to the uploader input.");
}
const files: File[] = Array.from(event.currentTarget.files);
return addData(dataset, files)
.then(() => {
console.log("Data added successfully.");
const onUpdate = (data: any) => {
onData({
nodes: data.payload.nodes,
links: data.payload.edges,
});
};
return cognifyDataset(dataset, onUpdate)
.then((data) => console.log(data));
});
};
return (
<div className="flex flex-col gap-4 mb-4">
{datasets.length ? datasets.map((dataset) => (
<div key={dataset.id} className="flex gap-8 items-center">
<div className="flex flex-row gap-4 items-center">
<StatusIndicator status={dataset.status} />
<span className="text-white">{dataset.name}</span>
</div>
<CTAButton type="button" className="relative">
<input type="file" multiple onChange={handleAddFiles.bind(null, dataset)} className="absolute w-full h-full cursor-pointer opacity-0" />
<span>+ Add Data</span>
</CTAButton>
</div>
)) : (
<CTAButton type="button" className="relative">
<input type="file" multiple onChange={handleAddFiles.bind(null, { name: "main_dataset" })} className="absolute w-full h-full cursor-pointer opacity-0" />
<span>+ Add Data</span>
</CTAButton>
)}
</div>
);
}

26
cognee-frontend/src/app/(graph)/CrewAITrigger.tsx Normal file
View file

@ -0,0 +1,26 @@
import { fetch } from "@/utils";
import { CTAButton, Input } from "@/ui/elements";
interface CrewAIFormPayload extends HTMLFormElement {
username1: HTMLInputElement;
username2: HTMLInputElement;
}
export default function CrewAITrigger() {
const handleRunCrewAI = (event: React.FormEvent<CrewAIFormPayload>) => {
fetch("/v1/crew-ai/run", {
method: "POST",
body: new FormData(event.currentTarget),
})
.then(response => response.json())
.then((data) => console.log(data));
};
return (
<form className="w-full flex flex-row gap-2 items-center" onSubmit={handleRunCrewAI}>
<Input type="text" placeholder="Github Username" required />
<Input type="text" placeholder="Github Username" required />
<CTAButton type="submit" className="whitespace-nowrap">Run CrewAI</CTAButton>
</form>
);
}

184
cognee-frontend/src/app/(graph)/GraphControls.tsx Normal file
View file

@ -0,0 +1,184 @@
"use client";
import { v4 as uuid4 } from "uuid";
import classNames from "classnames";
import { NodeObject } from "react-force-graph-2d";
import { ChangeEvent, useImperativeHandle, useState } from "react";
import { DeleteIcon } from "@/ui/Icons";
import { FeedbackForm } from "@/ui/Partials";
import { CTAButton, Input, NeutralButton, Select } from "@/ui/elements";
interface GraphControlsProps {
isAddNodeFormOpen: boolean;
ref: React.RefObject<GraphControlsAPI>;
onFitIntoView: () => void;
onGraphShapeChange: (shape: string) => void;
}
export interface GraphControlsAPI {
setSelectedNode: (node: NodeObject | null) => void;
getSelectedNode: () => NodeObject | null;
}
type ActivityLog = {
id: string;
timestamp: number;
activity: string;
}[];
type NodeProperties = {
id: string;
name: string;
value: string;
}[];
export default function GraphControls({ isAddNodeFormOpen, onGraphShapeChange, onFitIntoView, ref }: GraphControlsProps) {
const [selectedNode, setSelectedNode] = useState<NodeObject | null>(null);
const [activityLog, setActivityLog] = useState<ActivityLog>([]);
const [nodeProperties, setNodeProperties] = useState<NodeProperties>([]);
const [newProperty, setNewProperty] = useState<NodeProperties[0]>({
id: uuid4(),
name: "",
value: "",
});
const handlePropertyChange = (property: NodeProperties[0], property_key: string, event: ChangeEvent<HTMLInputElement>) => {
const value = event.target.value;
setNodeProperties(nodeProperties.map((nodeProperty) => (nodeProperty.id === property.id ? {...nodeProperty, [property_key]: value } : nodeProperty)));
};
const handlePropertyAdd = () => {
if (newProperty.name && newProperty.value) {
setNodeProperties([...nodeProperties, newProperty]);
setNewProperty({ id: uuid4(), name: "", value: "" });
} else {
alert("Please fill in both name and value fields for the new property.");
}
};
const handlePropertyDelete = (property: NodeProperties[0]) => {
setNodeProperties(nodeProperties.filter((nodeProperty) => nodeProperty.id !== property.id));
};
const handleNewPropertyChange = (property: NodeProperties[0], property_key: string, event: ChangeEvent<HTMLInputElement>) => {
const value = event.target.value;
setNewProperty({...property, [property_key]: value });
};
useImperativeHandle(ref, () => ({
setSelectedNode,
getSelectedNode: () => selectedNode,
}));
const [selectedTab, setSelectedTab] = useState("nodeDetails");
const handleGraphShapeControl = (event: ChangeEvent<HTMLSelectElement>) => {
onGraphShapeChange(event.target.value);
};
return (
<>
<div className="flex">
<button onClick={() => setSelectedTab("nodeDetails")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30", { "border-b-indigo-600 text-white": selectedTab === "nodeDetails" })}>
<span className="whitespace-nowrap">Node Details</span>
</button>
<button onClick={() => setSelectedTab("activityLog")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30", { "border-b-indigo-600 text-white": selectedTab === "activityLog" })}>
<span className="whitespace-nowrap">Activity Log</span>
</button>
<button onClick={() => setSelectedTab("feedback")} className={classNames("cursor-pointer pt-4 pb-4 align-center text-gray-300 border-b-2 w-30", { "border-b-indigo-600 text-white": selectedTab === "feedback" })}>
<span className="whitespace-nowrap">Feedback</span>
</button>
</div>
<div className="pt-4">
{selectedTab === "nodeDetails" && (
<>
<div className="w-full flex flex-row gap-2 items-center mb-4">
<label className="text-gray-300 whitespace-nowrap">Graph Shape:</label>
<Select onChange={handleGraphShapeControl}>
<option selected value="none">None</option>
<option value="td">Top-down</option>
<option value="bu">Bottom-up</option>
<option value="lr">Left-right</option>
<option value="rl">Right-left</option>
<option value="radialin">Radial-in</option>
<option value="radialout">Radial-out</option>
</Select>
</div>
<NeutralButton onClick={onFitIntoView} className="mb-4">Fit Graph into View</NeutralButton>
{isAddNodeFormOpen ? (
<form className="flex flex-col gap-4" onSubmit={() => {}}>
<div className="flex flex-row gap-4 items-center">
<span className="text-gray-300 whitespace-nowrap">Source Node ID:</span>
<Input readOnly type="text" defaultValue={selectedNode!.id} />
</div>
<div className="flex flex-col gap-4 items-end">
{nodeProperties.map((property) => (
<div key={property.id} className="w-full flex flex-row gap-2 items-center">
<Input className="flex-1/3" type="text" placeholder="Property name" required value={property.name} onChange={handlePropertyChange.bind(null, property, "name")} />
<Input className="flex-2/3" type="text" placeholder="Property value" required value={property.value} onChange={handlePropertyChange.bind(null, property, "value")} />
<button className="border-1 border-white p-2 rounded-sm" onClick={handlePropertyDelete.bind(null, property)}>
<DeleteIcon width={16} height={18} color="white" />
</button>
</div>
))}
<div className="w-full flex flex-row gap-2 items-center">
<Input className="flex-1/3" type="text" placeholder="Property name" required value={newProperty.name} onChange={handleNewPropertyChange.bind(null, newProperty, "name")} />
<Input className="flex-2/3" type="text" placeholder="Property value" required value={newProperty.value} onChange={handleNewPropertyChange.bind(null, newProperty, "value")} />
<NeutralButton type="button" className="" onClick={handlePropertyAdd}>Add</NeutralButton>
</div>
</div>
<CTAButton type="submit">Add Node</CTAButton>
</form>
) : (
selectedNode ? (
<div className="flex flex-col gap-4">
<div className="flex flex-col gap-2">
<div className="flex gap-2 items-center">
<span className="text-gray-300">ID:</span>
<span className="text-white">{selectedNode.id}</span>
</div>
{Object.entries(selectedNode.properties).map(([key, value]) => (
<div key={key} className="flex gap-2 items-center">
<span className="text-gray-300">{key}:</span>
<span className="text-white">{typeof value === "object" ? JSON.stringify(value) : value as string}</span>
</div>
))}
</div>
<CTAButton type="button" onClick={() => {}}>Edit Node</CTAButton>
</div>
) : (
<span className="text-white">No node selected.</span>
)
)}
</>
)}
{selectedTab === "activityLog" && (
<div className="flex flex-col gap-2">
{activityLog.map((activity) => (
<div key={activity.id} className="flex gap-2 items-center">
<span className="text-gray-300">{activity.timestamp}</span>
<span className="text-white">{activity.activity}</span>
</div>
))}
{!activityLog.length && <span className="text-white">No activity logged.</span>}
</div>
)}
{selectedTab === "feedback" && (
<div className="flex flex-col gap-2">
<FeedbackForm onSuccess={() => {}} />
</div>
)}
</div>
</>
);
}

276
cognee-frontend/src/app/(graph)/GraphView.tsx Normal file
View file

@ -0,0 +1,276 @@
"use client";
import { forceCollide, forceManyBody } from "d3-force-3d";
import { useEffect, useRef, useState } from "react";
import ForceGraph, { ForceGraphMethods, LinkObject, NodeObject } from "react-force-graph-2d";
import { TextLogo } from "@/ui/App";
import { Divider } from "@/ui/Layout";
import { Footer } from "@/ui/Partials";
import CrewAITrigger from "./CrewAITrigger";
import CogneeAddWidget, { NodesAndEdges } from "./CogneeAddWidget";
import GraphControls, { GraphControlsAPI } from "./GraphControls";
import { useBoolean } from "@/utils";
// import exampleData from "./example_data.json";
interface GraphNode {
id: string | number;
label: string;
properties?: {};
}
interface GraphData {
nodes: GraphNode[];
links: { source: string | number; target: string | number; label: string }[];
}
export default function GraphView() {
const {
value: isAddNodeFormOpen,
setTrue: enableAddNodeForm,
setFalse: disableAddNodeForm,
} = useBoolean(false);
const [data, updateData] = useState<GraphData | null>(null);
const onDataChange = (newData: NodesAndEdges) => {
if (data === null) {
updateData({
nodes: newData.nodes,
links: newData.links,
});
} else {
updateData({
nodes: [...data.nodes, ...newData.nodes],
links: [...data.links, ...newData.links],
});
}
};
const graphRef = useRef<ForceGraphMethods>();
const graphControls = useRef<GraphControlsAPI>(null);
const handleNodeClick = (node: NodeObject) => {
graphControls.current?.setSelectedNode(node);
graphRef.current?.d3ReheatSimulation();
};
const textSize = 6;
const nodeSize = 15;
const addNodeDistanceFromSourceNode = 15;
const handleBackgroundClick = (event: MouseEvent) => {
const graphBoundingBox = document.getElementById("graph-container")?.querySelector("canvas")?.getBoundingClientRect();
const x = event.clientX - graphBoundingBox!.x;
const y = event.clientY - graphBoundingBox!.y;
const graphClickCoords = graphRef.current!.screen2GraphCoords(x, y);
const selectedNode = graphControls.current?.getSelectedNode();
if (!selectedNode) {
return;
}
const distanceFromAddNode = Math.sqrt(
Math.pow(graphClickCoords.x - (selectedNode!.x! + addNodeDistanceFromSourceNode), 2)
+ Math.pow(graphClickCoords.y - (selectedNode!.y! + addNodeDistanceFromSourceNode), 2)
);
if (distanceFromAddNode <= 10) {
enableAddNodeForm();
} else {
disableAddNodeForm();
graphControls.current?.setSelectedNode(null);
}
};
function renderNode(node: NodeObject, ctx: CanvasRenderingContext2D, globalScale: number) {
const selectedNode = graphControls.current?.getSelectedNode();
ctx.save();
if (node.id === selectedNode?.id) {
ctx.fillStyle = "gray";
ctx.beginPath();
ctx.arc(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode, 10, 0, 2 * Math.PI);
ctx.fill();
ctx.beginPath();
ctx.moveTo(node.x! + addNodeDistanceFromSourceNode - 5, node.y! + addNodeDistanceFromSourceNode)
ctx.lineTo(node.x! + addNodeDistanceFromSourceNode - 5 + 10, node.y! + addNodeDistanceFromSourceNode);
ctx.stroke();
ctx.beginPath();
ctx.moveTo(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode - 5)
ctx.lineTo(node.x! + addNodeDistanceFromSourceNode, node.y! + addNodeDistanceFromSourceNode - 5 + 10);
ctx.stroke();
}
// ctx.beginPath();
// ctx.arc(node.x, node.y, nodeSize, 0, 2 * Math.PI);
// ctx.fill();
// draw text label (with background rect)
const textPos = {
x: node.x!,
y: node.y!,
};
ctx.translate(textPos.x, textPos.y);
ctx.textAlign = "center";
ctx.textBaseline = "middle";
ctx.fillStyle = "#333333";
ctx.font = `${textSize}px Sans-Serif`;
ctx.fillText(node.label, 0, 0);
ctx.restore();
}
function renderLink(link: LinkObject, ctx: CanvasRenderingContext2D) {
const MAX_FONT_SIZE = 4;
const LABEL_NODE_MARGIN = nodeSize * 1.5;
const start = link.source;
const end = link.target;
// ignore unbound links
if (typeof start !== "object" || typeof end !== "object") return;
const textPos = {
x: start.x! + (end.x! - start.x!) / 2,
y: start.y! + (end.y! - start.y!) / 2,
};
const relLink = { x: end.x! - start.x!, y: end.y! - start.y! };
const maxTextLength = Math.sqrt(Math.pow(relLink.x, 2) + Math.pow(relLink.y, 2)) - LABEL_NODE_MARGIN * 2;
let textAngle = Math.atan2(relLink.y, relLink.x);
// maintain label vertical orientation for legibility
if (textAngle > Math.PI / 2) textAngle = -(Math.PI - textAngle);
if (textAngle < -Math.PI / 2) textAngle = -(-Math.PI - textAngle);
const label = link.label
// estimate fontSize to fit in link length
ctx.font = "1px Sans-Serif";
const fontSize = Math.min(MAX_FONT_SIZE, maxTextLength / ctx.measureText(label).width);
ctx.font = `${fontSize}px Sans-Serif`;
const textWidth = ctx.measureText(label).width;
const bckgDimensions = [textWidth, fontSize].map(n => n + fontSize * 0.2); // some padding
// draw text label (with background rect)
ctx.save();
ctx.translate(textPos.x, textPos.y);
ctx.rotate(textAngle);
ctx.fillStyle = "rgba(255, 255, 255, 0.8)";
ctx.fillRect(- bckgDimensions[0] / 2, - bckgDimensions[1] / 2, bckgDimensions[0], bckgDimensions[1]);
ctx.textAlign = "center";
ctx.textBaseline = "middle";
ctx.fillStyle = "darkgrey";
ctx.fillText(label, 0, 0);
ctx.restore();
}
function handleDagError(loopNodeIds: (string | number)[]) {
console.log(loopNodeIds);
}
useEffect(() => {
// add collision force
graphRef.current!.d3Force("collision", forceCollide(nodeSize * 1.5));
graphRef.current!.d3Force("charge", forceManyBody().strength(-1500).distanceMin(300).distanceMax(900));
}, [data]);
const [graphShape, setGraphShape] = useState<string | undefined>(undefined);
return (
<main className="flex flex-col h-full">
<div className="pt-6 pr-3 pb-3 pl-6">
<TextLogo width={86} height={24} />
</div>
<Divider />
<div className="w-full h-full relative overflow-hidden">
<div className="w-full h-full" id="graph-container">
{data ? (
<ForceGraph
ref={graphRef}
dagMode={graphShape as undefined}
dagLevelDistance={300}
onDagError={handleDagError}
graphData={data}
nodeLabel="label"
nodeRelSize={nodeSize}
nodeCanvasObject={renderNode}
nodeCanvasObjectMode={() => "after"}
nodeAutoColorBy="group"
linkLabel="label"
linkCanvasObject={renderLink}
linkCanvasObjectMode={() => "after"}
linkDirectionalArrowLength={3.5}
linkDirectionalArrowRelPos={1}
onNodeClick={handleNodeClick}
onBackgroundClick={handleBackgroundClick}
d3VelocityDecay={0.3}
/>
) : (
<ForceGraph
ref={graphRef}
dagMode="lr"
dagLevelDistance={100}
graphData={{
nodes: [{ id: 1, label: "Add" }, { id: 2, label: "Cognify" }, { id: 3, label: "Search" }],
links: [{ source: 1, target: 2, label: "but don't forget to" }, { source: 2, target: 3, label: "and after that you can" }],
}}
nodeLabel="label"
nodeRelSize={20}
nodeCanvasObject={renderNode}
nodeCanvasObjectMode={() => "after"}
nodeAutoColorBy="group"
linkLabel="label"
linkCanvasObject={renderLink}
linkCanvasObjectMode={() => "after"}
linkDirectionalArrowLength={3.5}
linkDirectionalArrowRelPos={1}
/>
)}
</div>
<div className="absolute top-2 left-2 bg-gray-500 pt-4 pr-4 pb-4 pl-4 rounded-md max-w-2xl">
<CogneeAddWidget onData={onDataChange} />
<CrewAITrigger />
</div>
<div className="absolute top-2 right-2 bg-gray-500 pt-4 pr-4 pb-4 pl-4 rounded-md">
<GraphControls
ref={graphControls}
isAddNodeFormOpen={isAddNodeFormOpen}
onFitIntoView={() => graphRef.current?.zoomToFit(1000, 50)}
onGraphShapeChange={setGraphShape}
/>
</div>
</div>
<Divider />
<div className="pl-6 pr-6">
<Footer>
<div className="flex flex-row items-center gap-6">
<span>Nodes: {data?.nodes.length}</span>
<span>Edges: {data?.links.length}</span>
</div>
</Footer>
</div>
</main>
);
}

1376
cognee-frontend/src/app/(graph)/example_data.json Normal file
View file

File diff suppressed because it is too large Load diff

16
cognee-frontend/src/app/auth/AuthPage.module.css
View file

@ -1,16 +0,0 @@
.main {
display: flex;
flex-direction: row;
flex-direction: column;
padding: 0;
min-height: 100vh;
}
.authContainer {
flex: 1;
display: flex;
padding: 24px 0;
margin: 0 auto;
max-width: 440px;
width: 100%;
}

37
cognee-frontend/src/app/auth/AuthPage.tsx
View file

@ -1,29 +1,24 @@
import { Spacer, Stack, Text } from 'ohmy-ui';
import { TextLogo } from '@/ui/App';
import Footer from '@/ui/Partials/Footer/Footer';
import styles from './AuthPage.module.css';
import { Divider } from '@/ui/Layout';
import SignInForm from '@/ui/Partials/SignInForm/SignInForm';
import { TextLogo } from "@/ui/App";
import { Divider } from "@/ui/Layout";
import Footer from "@/ui/Partials/Footer/Footer";
import SignInForm from "@/ui/Partials/SignInForm/SignInForm";
export default function AuthPage() {
return (
<main className={styles.main}>
<Spacer inset vertical="2" horizontal="2">
<Stack orientation="horizontal" gap="between" align="center">
<TextLogo width={158} height={44} color="white" />
</Stack>
</Spacer>
<Divider />
<div className={styles.authContainer}>
<Stack gap="4" style={{ width: '100%' }}>
<h1><Text size="large">Sign in</Text></h1>
<SignInForm />
</Stack>
<main className="flex flex-col h-full">
<div className="pt-6 pr-3 pb-3 pl-6">
<TextLogo width={86} height={24} />
</div>
<Spacer inset horizontal="3" wrap>
<Divider />
<div className="w-full max-w-md pt-12 pb-6 m-auto">
<div className="flex flex-col w-full gap-8">
<h1><span className="text-xl">Sign in</span></h1>
<SignInForm />
</div>
</div>
<div className="pl-6 pr-6">
<Footer />
</Spacer>
</div>
</main>
)
}

13
cognee-frontend/src/app/globals.css
View file

@ -15,23 +15,16 @@
--textarea-default-color: #0D051C !important;
}
* {
box-sizing: border-box;
padding: 0;
margin: 0;
}
html,
body {
height: 100%;
max-width: 100vw;
overflow-x: hidden;
}
body {
background: var(--global-background-default);
}
a {
color: inherit;
text-decoration: none;
}
@import "tailwindcss";

130
cognee-frontend/src/app/page copy.tsx Normal file
View file

@ -0,0 +1,130 @@
'use client';
import { useCallback, useEffect, useState } from 'react';
import styles from "./page.module.css";
import { GhostButton, Notification, NotificationContainer, Spacer, Stack, Text, useBoolean, useNotifications } from 'ohmy-ui';
import useDatasets from '@/modules/ingestion/useDatasets';
import DataView, { Data } from '@/modules/ingestion/DataView';
import DatasetsView from '@/modules/ingestion/DatasetsView';
import classNames from 'classnames';
import addData from '@/modules/ingestion/addData';
import cognifyDataset from '@/modules/datasets/cognifyDataset';
import getDatasetData from '@/modules/datasets/getDatasetData';
import { Footer, SettingsModal } from '@/ui/Partials';
import { TextLogo } from '@/ui/App';
import { SettingsIcon } from '@/ui/Icons';
export default function Home() {
const {
datasets,
refreshDatasets,
} = useDatasets();
const [datasetData, setDatasetData] = useState<Data[]>([]);
const [selectedDataset, setSelectedDataset] = useState<string | null>(null);
useEffect(() => {
refreshDatasets();
}, [refreshDatasets]);
const openDatasetData = (dataset: { id: string }) => {
getDatasetData(dataset)
.then(setDatasetData)
.then(() => setSelectedDataset(dataset.id));
};
const closeDatasetData = () => {
setDatasetData([]);
setSelectedDataset(null);
};
const { notifications, showNotification } = useNotifications();
const onDataAdd = useCallback((dataset: { id: string }, files: File[]) => {
return addData(dataset, files)
.then(() => {
showNotification("Data added successfully. Please run \"Cognify\" when ready.", 5000);
openDatasetData(dataset);
});
}, [showNotification])
const onDatasetCognify = useCallback((dataset: { id: string, name: string }) => {
showNotification(`Cognification started for dataset "${dataset.name}".`, 5000);
return cognifyDataset(dataset)
.then(() => {
showNotification(`Dataset "${dataset.name}" cognified.`, 5000);
})
.catch(() => {
showNotification(`Dataset "${dataset.name}" cognification failed. Please try again.`, 5000);
});
}, [showNotification]);
const onCognify = useCallback(() => {
const dataset = datasets.find((dataset) => dataset.id === selectedDataset);
return onDatasetCognify({
id: dataset!.id,
name: dataset!.name,
});
}, [datasets, onDatasetCognify, selectedDataset]);
const {
value: isSettingsModalOpen,
setTrue: openSettingsModal,
setFalse: closeSettingsModal,
} = useBoolean(false);
return (
<main className={styles.main}>
<Spacer inset vertical="2" horizontal="2">
<Stack orientation="horizontal" gap="between" align="center">
<TextLogo width={158} height={44} color="white" />
<GhostButton hugContent onClick={openSettingsModal}>
<SettingsIcon />
</GhostButton>
</Stack>
</Spacer>
<SettingsModal isOpen={isSettingsModalOpen} onClose={closeSettingsModal} />
<Spacer inset vertical="1" horizontal="3">
<div className={styles.data}>
<div className={classNames(styles.datasetsView, {
[styles.openDatasetData]: datasetData.length > 0,
})}>
<DatasetsView
datasets={datasets}
onDatasetClick={openDatasetData}
onDatasetCognify={onDatasetCognify}
/>
</div>
{datasetData.length > 0 && selectedDataset && (
<div className={styles.dataView}>
<DataView
data={datasetData}
datasetId={selectedDataset}
onClose={closeDatasetData}
onDataAdd={onDataAdd}
onCognify={onCognify}
/>
</div>
)}
</div>
</Spacer>
<Spacer inset horizontal="3" wrap>
<Footer />
</Spacer>
<NotificationContainer gap="1" bottom right>
{notifications.map((notification, index: number) => (
<Notification
key={notification.id}
isOpen={notification.isOpen}
style={{ top: `${index * 60}px` }}
expireIn={notification.expireIn}
onClose={notification.delete}
>
<Text nowrap>{notification.message}</Text>
</Notification>
))}
</NotificationContainer>
</main>
);
}

131
cognee-frontend/src/app/page.tsx
View file

@ -1,130 +1 @@
'use client';
import { useCallback, useEffect, useState } from 'react';
import styles from "./page.module.css";
import { GhostButton, Notification, NotificationContainer, Spacer, Stack, Text, useBoolean, useNotifications } from 'ohmy-ui';
import useDatasets from '@/modules/ingestion/useDatasets';
import DataView, { Data } from '@/modules/ingestion/DataView';
import DatasetsView from '@/modules/ingestion/DatasetsView';
import classNames from 'classnames';
import addData from '@/modules/ingestion/addData';
import cognifyDataset from '@/modules/datasets/cognifyDataset';
import getDatasetData from '@/modules/datasets/getDatasetData';
import { Footer, SettingsModal } from '@/ui/Partials';
import { TextLogo } from '@/ui/App';
import { SettingsIcon } from '@/ui/Icons';
export default function Home() {
const {
datasets,
refreshDatasets,
} = useDatasets();
const [datasetData, setDatasetData] = useState<Data[]>([]);
const [selectedDataset, setSelectedDataset] = useState<string | null>(null);
useEffect(() => {
refreshDatasets();
}, [refreshDatasets]);
const openDatasetData = (dataset: { id: string }) => {
getDatasetData(dataset)
.then(setDatasetData)
.then(() => setSelectedDataset(dataset.id));
};
const closeDatasetData = () => {
setDatasetData([]);
setSelectedDataset(null);
};
const { notifications, showNotification } = useNotifications();
const onDataAdd = useCallback((dataset: { id: string }, files: File[]) => {
return addData(dataset, files)
.then(() => {
showNotification("Data added successfully. Please run \"Cognify\" when ready.", 5000);
openDatasetData(dataset);
});
}, [showNotification])
const onDatasetCognify = useCallback((dataset: { id: string, name: string }) => {
showNotification(`Cognification started for dataset "${dataset.name}".`, 5000);
return cognifyDataset(dataset)
.then(() => {
showNotification(`Dataset "${dataset.name}" cognified.`, 5000);
})
.catch(() => {
showNotification(`Dataset "${dataset.name}" cognification failed. Please try again.`, 5000);
});
}, [showNotification]);
const onCognify = useCallback(() => {
const dataset = datasets.find((dataset) => dataset.id === selectedDataset);
return onDatasetCognify({
id: dataset!.id,
name: dataset!.name,
});
}, [datasets, onDatasetCognify, selectedDataset]);
const {
value: isSettingsModalOpen,
setTrue: openSettingsModal,
setFalse: closeSettingsModal,
} = useBoolean(false);
return (
<main className={styles.main}>
<Spacer inset vertical="2" horizontal="2">
<Stack orientation="horizontal" gap="between" align="center">
<TextLogo width={158} height={44} color="white" />
<GhostButton hugContent onClick={openSettingsModal}>
<SettingsIcon />
</GhostButton>
</Stack>
</Spacer>
<SettingsModal isOpen={isSettingsModalOpen} onClose={closeSettingsModal} />
<Spacer inset vertical="1" horizontal="3">
<div className={styles.data}>
<div className={classNames(styles.datasetsView, {
[styles.openDatasetData]: datasetData.length > 0,
})}>
<DatasetsView
datasets={datasets}
onDatasetClick={openDatasetData}
onDatasetCognify={onDatasetCognify}
/>
</div>
{datasetData.length > 0 && selectedDataset && (
<div className={styles.dataView}>
<DataView
data={datasetData}
datasetId={selectedDataset}
onClose={closeDatasetData}
onDataAdd={onDataAdd}
onCognify={onCognify}
/>
</div>
)}
</div>
</Spacer>
<Spacer inset horizontal="3" wrap>
<Footer />
</Spacer>
<NotificationContainer gap="1" bottom right>
{notifications.map((notification, index: number) => (
<Notification
key={notification.id}
isOpen={notification.isOpen}
style={{ top: `${index * 60}px` }}
expireIn={notification.expireIn}
onClose={notification.delete}
>
<Text nowrap>{notification.message}</Text>
</Notification>
))}
</NotificationContainer>
</main>
);
}
export { default } from "./(graph)/GraphView";

34
cognee-frontend/src/modules/datasets/cognifyDataset.ts
View file

@ -1,6 +1,6 @@
import { fetch } from '@/utils';
export default function cognifyDataset(dataset: { id?: string, name?: string }) {
export default function cognifyDataset(dataset: { id?: string, name?: string }, onUpdate = (data: []) => {}) {
return fetch('/v1/cognify', {
method: 'POST',
headers: {
@ -9,5 +9,35 @@ export default function cognifyDataset(dataset: { id?: string, name?: string })
body: JSON.stringify({
datasets: [dataset.id || dataset.name],
}),
}).then((response) => response.json());
})
.then((response) => response.json())
.then((data) => {
const websocket = new WebSocket(`ws://localhost:8000/api/v1/cognify/subscribe/${data.pipeline_run_id}`);
websocket.onopen = () => {
websocket.send(JSON.stringify({
"Authorization": `Bearer ${localStorage.getItem("access_token")}`,
}));
};
let isCognifyDone = false;
websocket.onmessage = (event) => {
const data = JSON.parse(event.data);
onUpdate(data);
if (data.status === "PipelineRunCompleted") {
isCognifyDone = true;
websocket.close();
}
};
return new Promise(async (resolve) => {
while (!isCognifyDone) {
await new Promise(resolve => setTimeout(resolve, 1000));
}
resolve(true);
});
});
}

6
cognee-frontend/src/modules/datasets/getDatasetGraph.ts Normal file
View file

@ -0,0 +1,6 @@
import { fetch } from '@/utils';
export default function getDatasetGraph(dataset: { id: string }) {
return fetch(`/v1/datasets/${dataset.id}/graph`)
.then((response) => response.json());
}

2
cognee-frontend/src/modules/ingestion/DatasetsView/DatasetsView.tsx
View file

@ -1,7 +1,7 @@
import { useState } from 'react';
import Link from 'next/link';
import { Explorer } from '@/ui/Partials';
import StatusIcon from './StatusIcon';
import StatusIcon from '@/ui/elements/StatusIndicator';
import { LoadingIndicator } from '@/ui/App';
import { DropdownMenu, GhostButton, Stack, Text, CTAButton, useBoolean, Modal, Spacer } from "ohmy-ui";
import styles from "./DatasetsView.module.css";

15
cognee-frontend/src/modules/ingestion/DatasetsView/StatusIcon.tsx
View file

@ -1,15 +0,0 @@
export default function StatusIcon({ status }: { status: 'DATASET_PROCESSING_COMPLETED' | string }) {
const isSuccess = status === 'DATASET_PROCESSING_COMPLETED';
return (
<div
style={{
width: '16px',
height: '16px',
borderRadius: '4px',
background: isSuccess ? '#53ff24' : '#ff5024',
}}
title={isSuccess ? 'Dataset cognified' : 'Cognify data in order to explore it'}
/>
);
}

6
cognee-frontend/src/modules/ingestion/useDatasets.ts
View file

@ -73,7 +73,7 @@ function useDatasets() {
}, []);
const fetchDatasets = useCallback(() => {
fetch('/v1/datasets', {
return fetch('/v1/datasets', {
headers: {
Authorization: `Bearer ${localStorage.getItem('access_token')}`,
},
@ -84,9 +84,9 @@ function useDatasets() {
if (datasets.length > 0) {
checkDatasetStatuses(datasets);
} else {
window.location.href = '/wizard';
}
return datasets;
})
.catch((error) => {
console.error('Error fetching datasets:', error);

6
cognee-frontend/src/ui/App/Loading/DefaultLoadingIndicator/LoadingIndicator.module.css
View file

@ -1,9 +1,9 @@
.loadingIndicator {
width: 16px;
height: 16px;
width: 1rem;
height: 1rem;
border-radius: 50%;
border: 2px solid var(--global-color-primary);
border: 0.18rem solid white;
border-top-color: transparent;
border-bottom-color: transparent;
animation: spin 2s linear infinite;

7
cognee-frontend/src/ui/Icons/DeleteIcon.tsx Normal file
View file

@ -0,0 +1,7 @@
export default function DeleteIcon({ width = 12, height = 14, color = 'currentColor' }) {
return (
<svg width={width} height={height} viewBox="0 0 12 14" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M3.625 1.87357H3.5C3.56875 1.87357 3.625 1.81732 3.625 1.74857V1.87357H8.375V1.74857C8.375 1.81732 8.43125 1.87357 8.5 1.87357H8.375V2.99857H9.5V1.74857C9.5 1.197 9.05156 0.748566 8.5 0.748566H3.5C2.94844 0.748566 2.5 1.197 2.5 1.74857V2.99857H3.625V1.87357ZM11.5 2.99857H0.5C0.223438 2.99857 0 3.222 0 3.49857V3.99857C0 4.06732 0.05625 4.12357 0.125 4.12357H1.06875L1.45469 12.2954C1.47969 12.8283 1.92031 13.2486 2.45313 13.2486H9.54688C10.0813 13.2486 10.5203 12.8298 10.5453 12.2954L10.9313 4.12357H11.875C11.9438 4.12357 12 4.06732 12 3.99857V3.49857C12 3.222 11.7766 2.99857 11.5 2.99857ZM9.42656 12.1236H2.57344L2.19531 4.12357H9.80469L9.42656 12.1236Z" fill={color} />
</svg>
);
}

2
cognee-frontend/src/ui/Icons/GitHubIcon.tsx
View file

@ -3,7 +3,7 @@ export default function GitHubIcon({ width = 24, height = 24, color = 'currentCo
<svg xmlns="http://www.w3.org/2000/svg" width={width} height={height} viewBox="0 0 28 28" className={className}>
<g transform="translate(-1477 -38)">
<rect width="28" height="28" transform="translate(1477 38)" fill={color} opacity="0" />
<path d="M16.142,1.9A13.854,13.854,0,0,0,11.78,28.966c.641.128,1.155-.577,1.155-1.154v-1.86c-3.848.834-5.067-1.86-5.067-1.86a4.169,4.169,0,0,0-1.411-2.052c-1.283-.9.064-.834.064-.834a2.758,2.758,0,0,1,2.117,1.283c1.09,1.86,3.528,1.668,4.3,1.347a3.463,3.463,0,0,1,.321-1.86c-4.361-.77-6.735-3.335-6.735-6.8A6.863,6.863,0,0,1,8.381,10.3a3.977,3.977,0,0,1,.192-4.1,5.708,5.708,0,0,1,4.1,1.86,9.685,9.685,0,0,1,3.463-.513,10.968,10.968,0,0,1,3.463.449,5.773,5.773,0,0,1,4.1-1.8,4.169,4.169,0,0,1,.257,4.1,6.863,6.863,0,0,1,1.8,4.875c0,3.463-2.373,6.029-6.735,6.8a3.464,3.464,0,0,1,.321,1.86v3.977a1.155,1.155,0,0,0,1.219,1.155A13.918,13.918,0,0,0,16.142,1.9Z" transform="translate(1474.913 36.102)" fill="#fdfdfd"/>
<path d="M16.142,1.9A13.854,13.854,0,0,0,11.78,28.966c.641.128,1.155-.577,1.155-1.154v-1.86c-3.848.834-5.067-1.86-5.067-1.86a4.169,4.169,0,0,0-1.411-2.052c-1.283-.9.064-.834.064-.834a2.758,2.758,0,0,1,2.117,1.283c1.09,1.86,3.528,1.668,4.3,1.347a3.463,3.463,0,0,1,.321-1.86c-4.361-.77-6.735-3.335-6.735-6.8A6.863,6.863,0,0,1,8.381,10.3a3.977,3.977,0,0,1,.192-4.1,5.708,5.708,0,0,1,4.1,1.86,9.685,9.685,0,0,1,3.463-.513,10.968,10.968,0,0,1,3.463.449,5.773,5.773,0,0,1,4.1-1.8,4.169,4.169,0,0,1,.257,4.1,6.863,6.863,0,0,1,1.8,4.875c0,3.463-2.373,6.029-6.735,6.8a3.464,3.464,0,0,1,.321,1.86v3.977a1.155,1.155,0,0,0,1.219,1.155A13.918,13.918,0,0,0,16.142,1.9Z" transform="translate(1474.913 36.102)" fill={color}/>
</g>
</svg>
);

1
cognee-frontend/src/ui/Icons/index.ts
View file

@ -1,3 +1,4 @@
export { default as DeleteIcon } from './DeleteIcon';
export { default as GithubIcon } from './GitHubIcon';
export { default as DiscordIcon } from './DiscordIcon';
export { default as SettingsIcon } from './SettingsIcon';

66
cognee-frontend/src/ui/Partials/FeedbackForm.tsx Normal file
View file

@ -0,0 +1,66 @@
"use client";
import { useState } from "react";
import { LoadingIndicator } from "@/ui/App";
import { fetch, useBoolean } from "@/utils";
import { CTAButton, TextArea } from "@/ui/elements";
interface SignInFormPayload extends HTMLFormElement {
feedback: HTMLTextAreaElement;
}
interface FeedbackFormProps {
onSuccess: () => void;
}
export default function FeedbackForm({ onSuccess }: FeedbackFormProps) {
const {
value: isSubmittingFeedback,
setTrue: disableFeedbackSubmit,
setFalse: enableFeedbackSubmit,
} = useBoolean(false);
const [feedbackError, setFeedbackError] = useState<string | null>(null);
const signIn = (event: React.FormEvent<SignInFormPayload>) => {
event.preventDefault();
const formElements = event.currentTarget;
const authCredentials = new FormData();
authCredentials.append("feedback", formElements.feedback.value);
setFeedbackError(null);
disableFeedbackSubmit();
fetch("/v1/feedback/reasoning", {
method: "POST",
body: authCredentials,
})
.then(response => response.json())
.then(() => {
onSuccess();
})
.catch(error => setFeedbackError(error.detail))
.finally(() => enableFeedbackSubmit());
};
return (
<form onSubmit={signIn} className="flex flex-col gap-2">
<div className="flex flex-col gap-2">
<div className="mb-4">
<label className="block text-white" htmlFor="feedback">Your feedback on agents reasoning</label>
<TextArea id="feedback" name="feedback" type="text" placeholder="Your feedback" />
</div>
</div>
<CTAButton type="submit">
<span>Submit feedback</span>
{isSubmittingFeedback && <LoadingIndicator />}
</CTAButton>
{feedbackError && (
<span className="text-s text-white">{feedbackError}</span>
)}
</form>
)
}

16
cognee-frontend/src/ui/Partials/Footer/Footer.module.css
View file

@ -1,16 +0,0 @@
.footer {
padding: 24px 0;
}
.leftSide {
display: flex;
flex-direction: column;
gap: 12px;
}
.rightSide {
display: flex;
flex-direction: row;
align-items: center;
gap: 24px;
}

40
cognee-frontend/src/ui/Partials/Footer/Footer.tsx
View file

@ -1,25 +1,25 @@
import Link from 'next/link';
import { Stack } from 'ohmy-ui';
import { DiscordIcon, GithubIcon } from '@/ui/Icons';
// import { TextLogo } from '@/ui/App';
import styles from './Footer.module.css';
import Link from "next/link";
import { DiscordIcon, GithubIcon } from "@/ui/Icons";
export default function Footer() {
interface FooterProps {
children?: React.ReactNode;
}
export default function Footer({ children }: FooterProps) {
return (
<footer className={styles.footer}>
<Stack orientation="horizontal" gap="between">
<div className={styles.leftSide}>
{/* <TextLogo width={92} height={24} /> */}
</div>
<div className={styles.rightSide}>
<Link target="_blank" href="https://github.com/topoteretes/cognee">
<GithubIcon color="white" />
</Link>
<Link target="_blank" href="https://discord.gg/m63hxKsp4p">
<DiscordIcon color="white" />
</Link>
</div>
</Stack>
<footer className="pt-6 pb-6 flex flex-row items-center justify-between">
<div>
{children}
</div>
<div className="flex flex-row gap-4">
<Link target="_blank" href="https://github.com/topoteretes/cognee">
<GithubIcon color="black" />
</Link>
<Link target="_blank" href="https://discord.gg/m63hxKsp4p">
<DiscordIcon color="black" />
</Link>
</div>
</footer>
);
}

75
cognee-frontend/src/ui/Partials/SignInForm/SignInForm.tsx
View file

@ -1,19 +1,9 @@
"use client";
import {
CTAButton,
FormGroup,
FormInput,
FormLabel,
Input,
Spacer,
Stack,
Text,
useBoolean,
} from 'ohmy-ui';
import { LoadingIndicator } from '@/ui/App';
import { fetch, handleServerErrors } from '@/utils';
import { useState } from 'react';
import { useState } from "react";
import { LoadingIndicator } from "@/ui/App";
import { fetch, useBoolean } from "@/utils";
import { CTAButton, Input } from "@/ui/elements";
interface SignInFormPayload extends HTMLFormElement {
vectorDBUrl: HTMLInputElement;
@ -22,10 +12,10 @@ interface SignInFormPayload extends HTMLFormElement {
}
const errorsMap = {
LOGIN_BAD_CREDENTIALS: 'Invalid username or password',
LOGIN_BAD_CREDENTIALS: "Invalid username or password",
};
export default function SignInForm({ onSignInSuccess = () => window.location.href = '/', submitButtonText = 'Sign in' }) {
export default function SignInForm({ onSignInSuccess = () => window.location.href = "/", submitButtonText = "Sign in" }) {
const {
value: isSigningIn,
setTrue: disableSignIn,
@ -46,14 +36,13 @@ export default function SignInForm({ onSignInSuccess = () => window.location.hre
setSignInError(null);
disableSignIn();
fetch('/v1/auth/login', {
method: 'POST',
fetch("/v1/auth/login", {
method: "POST",
body: authCredentials,
})
.then(handleServerErrors)
.then(response => response.json())
.then((bearer) => {
window.localStorage.setItem('access_token', bearer.access_token);
window.localStorage.setItem("access_token", bearer.access_token);
onSignInSuccess();
})
.catch(error => setSignInError(errorsMap[error.detail as keyof typeof errorsMap]))
@ -61,36 +50,26 @@ export default function SignInForm({ onSignInSuccess = () => window.location.hre
};
return (
<form onSubmit={signIn} style={{ width: '100%' }}>
<Stack gap="4" orientation="vertical">
<Stack gap="4" orientation="vertical">
<FormGroup orientation="vertical" align="center/" gap="2">
<FormLabel>Email:</FormLabel>
<FormInput>
<Input defaultValue="default_user@example.com" name="email" type="email" placeholder="Your email address" />
</FormInput>
</FormGroup>
<FormGroup orientation="vertical" align="center/" gap="2">
<FormLabel>Password:</FormLabel>
<FormInput>
<Input defaultValue="default_password" name="password" type="password" placeholder="Your password" />
</FormInput>
</FormGroup>
</Stack>
<form onSubmit={signIn} className="flex flex-col gap-2">
<div className="flex flex-col gap-2">
<div className="mb-4">
<label className="block mb-2" htmlFor="email">Email</label>
<Input id="email" defaultValue="default_user@example.com" name="email" type="email" placeholder="Your email address" />
</div>
<div className="mb-4">
<label className="block mb-2" htmlFor="password">Password</label>
<Input id="password" defaultValue="default_password" name="password" type="password" placeholder="Your password" />
</div>
</div>
<Spacer top="2">
<CTAButton type="submit">
<Stack gap="2" orientation="horizontal" align="/center">
{submitButtonText}
{isSigningIn && <LoadingIndicator />}
</Stack>
</CTAButton>
</Spacer>
<CTAButton type="submit">
{submitButtonText}
{isSigningIn && <LoadingIndicator />}
</CTAButton>
{signInError && (
<Text>{signInError}</Text>
)}
</Stack>
{signInError && (
<span className="text-s text-white">{signInError}</span>
)}
</form>
)
}

11
cognee-frontend/src/ui/Partials/index.ts
View file

@ -1,5 +1,6 @@
export { default as Footer } from './Footer/Footer';
export { default as SettingsModal } from './SettingsModal/SettingsModal';
export { default as SearchView } from './SearchView/SearchView';
export { default as IFrameView } from './IFrameView/IFrameView';
export { default as Explorer } from './Explorer/Explorer';
export { default as Footer } from "./Footer/Footer";
export { default as SettingsModal } from "./SettingsModal/SettingsModal";
export { default as SearchView } from "./SearchView/SearchView";
export { default as IFrameView } from "./IFrameView/IFrameView";
export { default as Explorer } from "./Explorer/Explorer";
export { default as FeedbackForm } from "./FeedbackForm";

8
cognee-frontend/src/ui/elements/CTAButton.tsx Normal file
View file

@ -0,0 +1,8 @@
import classNames from 'classnames';
import { ButtonHTMLAttributes } from "react";
export default function CTAButton({ children, className, ...props }: ButtonHTMLAttributes<HTMLButtonElement>) {
return (
<button className={classNames("flex flex-row justify-center items-center gap-2 cursor-pointer rounded-md bg-indigo-600 px-3 py-2 text-sm font-semibold text-white shadow-xs hover:bg-indigo-500 focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600", className)} {...props}>{children}</button>
);
}

8
cognee-frontend/src/ui/elements/Input.tsx Normal file
View file

@ -0,0 +1,8 @@
import classNames from "classnames"
import { InputHTMLAttributes } from "react"
export default function Input({ className, ...props }: InputHTMLAttributes<HTMLInputElement>) {
return (
<input className={classNames("block w-full rounded-md bg-white px-3 py-1.5 text-base text-gray-900 outline-1 -outline-offset-1 outline-gray-300 placeholder:text-gray-400 focus:outline-2 focus:-outline-offset-2 focus:outline-indigo-600 sm:text-sm/6", className)} {...props} />
)
}

8
cognee-frontend/src/ui/elements/NeutralButton.tsx Normal file
View file

@ -0,0 +1,8 @@
import classNames from 'classnames';
import { ButtonHTMLAttributes } from "react";
export default function CTAButton({ children, className, ...props }: ButtonHTMLAttributes<HTMLButtonElement>) {
return (
<button className={classNames("flex flex-row justify-center items-center gap-2 cursor-pointer rounded-md bg-transparent px-3 py-2 text-sm font-semibold text-white shadow-xs border-1 border-white hover:bg-gray-400 focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600", className)} {...props}>{children}</button>
);
}

10
cognee-frontend/src/ui/elements/Select.tsx Normal file
View file

@ -0,0 +1,10 @@
import classNames from "classnames";
import { SelectHTMLAttributes } from "react";
export default function Select({ children, className, ...props }: SelectHTMLAttributes<HTMLSelectElement>) {
return (
<select className={classNames("block w-full appearance-none rounded-md bg-white py-1.5 pr-8 pl-3 text-base text-gray-900 outline-1 -outline-offset-1 outline-gray-300 focus:outline-2 focus:-outline-offset-2 focus:outline-indigo-600 sm:text-sm/6", className)} {...props}>
{children}
</select>
);
}

22
cognee-frontend/src/ui/elements/StatusIndicator.tsx Normal file
View file

@ -0,0 +1,22 @@
export default function StatusIndicator({ status }: { status: "DATASET_PROCESSING_COMPLETED" | string }) {
const statusColor = {
DATASET_PROCESSING_STARTED: "#ffd500",
DATASET_PROCESSING_INITIATED: "#ffd500",
DATASET_PROCESSING_COMPLETED: "#53ff24",
DATASET_PROCESSING_ERRORED: "#ff5024",
};
const isSuccess = status === "DATASET_PROCESSING_COMPLETED";
return (
<div
style={{
width: "16px",
height: "16px",
borderRadius: "4px",
background: statusColor[status as keyof typeof statusColor],
}}
title={isSuccess ? "Dataset cognified" : "Cognify data in order to explore it"}
/>
);
}

7
cognee-frontend/src/ui/elements/TextArea.tsx Normal file
View file

@ -0,0 +1,7 @@
import { InputHTMLAttributes } from "react"
export default function TextArea(props: InputHTMLAttributes<HTMLTextAreaElement>) {
return (
<textarea className="block w-full mt-2 rounded-md bg-white px-3 py-1.5 text-base text-gray-900 outline-1 -outline-offset-1 outline-gray-300 placeholder:text-gray-400 focus:outline-2 focus:-outline-offset-2 focus:outline-indigo-600 sm:text-sm/6" {...props} />
)
}

6
cognee-frontend/src/ui/elements/index.ts Normal file
View file

@ -0,0 +1,6 @@
export { default as Input } from "./Input";
export { default as Select } from "./Select";
export { default as TextArea } from "./TextArea";
export { default as CTAButton } from "./CTAButton";
export { default as NeutralButton } from "./NeutralButton";
export { default as StatusIndicator } from "./StatusIndicator";

5
cognee-frontend/src/utils/index.ts
View file

@ -1,2 +1,3 @@
export { default as fetch } from './fetch';
export { default as handleServerErrors } from './handleServerErrors';
export { default as fetch } from "./fetch";
export { default as handleServerErrors } from "./handleServerErrors";
export { default as useBoolean } from "./useBoolean";

14
cognee-frontend/src/utils/useBoolean.ts Normal file
View file

@ -0,0 +1,14 @@
import { useState } from "react";
export default function useBoolean(initialValue: boolean) {
const [value, setValue] = useState(initialValue);
const setTrue = () => setValue(true);
const setFalse = () => setValue(false);
return {
value,
setTrue,
setFalse,
};
}

15
cognee/api/client.py
View file

@ -3,11 +3,18 @@
import os
import uvicorn
from cognee.shared.logging_utils import get_logger, setup_logging
import sentry_sdk
from traceback import format_exc
from contextlib import asynccontextmanager
from fastapi import Request
from fastapi import FastAPI, status
from fastapi.encoders import jsonable_encoder
from fastapi.responses import JSONResponse, Response
from fastapi.middleware.cors import CORSMiddleware
from fastapi.exceptions import RequestValidationError
from cognee.exceptions import CogneeApiError
from cognee.shared.logging_utils import get_logger, setup_logging
from cognee.api.v1.permissions.routers import get_permissions_router
from cognee.api.v1.settings.routers import get_settings_router
from cognee.api.v1.datasets.routers import get_datasets_router
@ -16,11 +23,6 @@ from cognee.api.v1.search.routers import get_search_router
from cognee.api.v1.add.routers import get_add_router
from cognee.api.v1.delete.routers import get_delete_router
from cognee.api.v1.responses.routers import get_responses_router
from fastapi import Request
from fastapi.encoders import jsonable_encoder
from fastapi.exceptions import RequestValidationError
from cognee.exceptions import CogneeApiError
from traceback import format_exc
from cognee.api.v1.users.routers import (
get_auth_router,
get_register_router,
@ -29,7 +31,6 @@ from cognee.api.v1.users.routers import (
get_users_router,
get_visualize_router,
)
from contextlib import asynccontextmanager
logger = get_logger()

9
cognee/api/v1/add/add.py
View file

@ -21,7 +21,9 @@ async def add(
Task(ingest_data, dataset_name, user, node_set, dataset_id),
]
await cognee_pipeline(
pipeline_run_info = None
async for run_info in cognee_pipeline(
tasks=tasks,
datasets=dataset_id if dataset_id else dataset_name,
data=data,
@ -29,4 +31,7 @@ async def add(
pipeline_name="add_pipeline",
vector_db_config=vector_db_config,
graph_db_config=graph_db_config,
)
):
pipeline_run_info = run_info
return pipeline_run_info

7
cognee/api/v1/add/routers/get_add_router.py
View file

@ -5,7 +5,6 @@ from fastapi.responses import JSONResponse
from fastapi import APIRouter
from typing import List, Optional
import subprocess
from cognee.modules.data.methods import get_dataset
from cognee.shared.logging_utils import get_logger
import requests
@ -18,7 +17,7 @@ logger = get_logger()
def get_add_router() -> APIRouter:
router = APIRouter()
@router.post("/", response_model=None)
@router.post("/", response_model=dict)
async def add(
data: List[UploadFile],
datasetName: str,
@ -51,7 +50,9 @@ def get_add_router() -> APIRouter:
# TODO: Update add call with dataset info
return await cognee_add(file_data)
else:
await cognee_add(data, dataset_name=datasetName, user=user, dataset_id=datasetId)
add_run = await cognee_add(data, datasetName, user=user, dataset_id=datasetId)
return add_run.model_dump()
except Exception as error:
return JSONResponse(status_code=409, content={"error": str(error)})

90
cognee/api/v1/cognify/cognify.py
View file

@ -1,13 +1,19 @@
import asyncio
from cognee.shared.logging_utils import get_logger
from typing import Union, Optional
from pydantic import BaseModel
from typing import Union, Optional
from cognee.infrastructure.llm import get_max_chunk_tokens
from cognee.modules.ontology.rdf_xml.OntologyResolver import OntologyResolver
from cognee.modules.pipelines.tasks.task import Task
from cognee.modules.users.models import User
from cognee.shared.logging_utils import get_logger
from cognee.shared.data_models import KnowledgeGraph
from cognee.infrastructure.llm import get_max_chunk_tokens
from cognee.modules.pipelines import cognee_pipeline
from cognee.modules.pipelines.tasks.task import Task
from cognee.modules.chunking.TextChunker import TextChunker
from cognee.modules.ontology.rdf_xml.OntologyResolver import OntologyResolver
from cognee.modules.pipelines.models.PipelineRunInfo import PipelineRunCompleted
from cognee.modules.pipelines.queues.pipeline_run_info_queues import push_to_queue
from cognee.modules.users.models import User
from cognee.tasks.documents import (
check_permissions_on_dataset,
classify_documents,
@ -16,8 +22,6 @@ from cognee.tasks.documents import (
from cognee.tasks.graph import extract_graph_from_data
from cognee.tasks.storage import add_data_points
from cognee.tasks.summarization import summarize_text
from cognee.modules.chunking.TextChunker import TextChunker
from cognee.modules.pipelines import cognee_pipeline
logger = get_logger("cognify")
@ -33,18 +37,84 @@ async def cognify(
ontology_file_path: Optional[str] = None,
vector_db_config: dict = None,
graph_db_config: dict = None,
run_in_background: bool = False,
):
tasks = await get_default_tasks(user, graph_model, chunker, chunk_size, ontology_file_path)
return await cognee_pipeline(
if run_in_background:
return await run_cognify_as_background_process(
tasks=tasks,
user=user,
datasets=datasets,
vector_db_config=vector_db_config,
graph_db_config=graph_db_config,
)
else:
return await run_cognify_blocking(
tasks=tasks,
user=user,
datasets=datasets,
vector_db_config=vector_db_config,
graph_db_config=graph_db_config,
)
async def run_cognify_blocking(
tasks,
user,
datasets,
graph_db_config: dict = None,
vector_db_config: dict = False,
):
pipeline_run_info = None
async for run_info in cognee_pipeline(
tasks=tasks,
datasets=datasets,
user=user,
pipeline_name="cognify_pipeline",
vector_db_config=vector_db_config,
graph_db_config=graph_db_config,
vector_db_config=vector_db_config,
):
pipeline_run_info = run_info
return pipeline_run_info
async def run_cognify_as_background_process(
tasks,
user,
datasets,
graph_db_config: dict = None,
vector_db_config: dict = False,
):
pipeline_run = cognee_pipeline(
tasks=tasks,
user=user,
datasets=datasets,
pipeline_name="cognify_pipeline",
graph_db_config=graph_db_config,
vector_db_config=vector_db_config,
)
pipeline_run_started_info = await anext(pipeline_run)
async def handle_rest_of_the_run():
while True:
try:
pipeline_run_info = await anext(pipeline_run)
push_to_queue(pipeline_run_info.pipeline_run_id, pipeline_run_info)
if isinstance(pipeline_run_info, PipelineRunCompleted):
break
except StopAsyncIteration:
break
asyncio.create_task(handle_rest_of_the_run())
return pipeline_run_started_info
async def get_default_tasks( # TODO: Find out a better way to do this (Boris's comment)
user: User = None,

120
cognee/api/v1/cognify/routers/get_cognify_router.py
View file

@ -1,12 +1,21 @@
import asyncio
from uuid import UUID
from typing import List, Optional
from pydantic import BaseModel
from fastapi import Depends
from fastapi import APIRouter
from typing import List, Optional
from fastapi.responses import JSONResponse
from fastapi import APIRouter, WebSocket, Depends, WebSocketDisconnect
from starlette.status import WS_1000_NORMAL_CLOSURE, WS_1008_POLICY_VIOLATION
from cognee.modules.users.models import User
from cognee.modules.users.methods import get_authenticated_user
from cognee.shared.data_models import KnowledgeGraph
from cognee.modules.users.methods import get_authenticated_user
from cognee.modules.pipelines.models.PipelineRunInfo import PipelineRunCompleted, PipelineRunInfo
from cognee.modules.graph.utils import deduplicate_nodes_and_edges, get_graph_from_model
from cognee.modules.pipelines.queues.pipeline_run_info_queues import (
get_from_queue,
initialize_queue,
remove_queue,
)
class CognifyPayloadDTO(BaseModel):
@ -24,10 +33,109 @@ def get_cognify_router() -> APIRouter:
from cognee.api.v1.cognify import cognify as cognee_cognify
try:
# Send dataset UUIDs if they are given, if not send dataset names
datasets = payload.dataset_ids if payload.dataset_ids else payload.datasets
await cognee_cognify(datasets, user, payload.graph_model)
cognify_run = await cognee_cognify(
datasets, user, payload.graph_model, run_in_background=True
)
return cognify_run.model_dump()
except Exception as error:
return JSONResponse(status_code=409, content={"error": str(error)})
@router.websocket("/subscribe/{pipeline_run_id}")
async def subscribe_to_cognify_info(websocket: WebSocket, pipeline_run_id: str):
await websocket.accept()
auth_message = await websocket.receive_json()
try:
await get_authenticated_user(auth_message.get("Authorization"))
except Exception:
await websocket.close(code=WS_1008_POLICY_VIOLATION, reason="Unauthorized")
return
pipeline_run_id = UUID(pipeline_run_id)
initialize_queue(pipeline_run_id)
while True:
pipeline_run_info = get_from_queue(pipeline_run_id)
if not pipeline_run_info:
await asyncio.sleep(2)
continue
if not isinstance(pipeline_run_info, PipelineRunInfo):
continue
try:
await websocket.send_json(
{
"pipeline_run_id": str(pipeline_run_info.pipeline_run_id),
"status": pipeline_run_info.status,
"payload": await get_nodes_and_edges(pipeline_run_info.payload)
if pipeline_run_info.payload
else None,
}
)
if isinstance(pipeline_run_info, PipelineRunCompleted):
remove_queue(pipeline_run_id)
await websocket.close(code=WS_1000_NORMAL_CLOSURE)
break
except WebSocketDisconnect:
remove_queue(pipeline_run_id)
break
return router
async def get_nodes_and_edges(data_points):
nodes = []
edges = []
added_nodes = {}
added_edges = {}
visited_properties = {}
results = await asyncio.gather(
*[
get_graph_from_model(
data_point,
added_nodes=added_nodes,
added_edges=added_edges,
visited_properties=visited_properties,
)
for data_point in data_points
]
)
for result_nodes, result_edges in results:
nodes.extend(result_nodes)
edges.extend(result_edges)
nodes, edges = deduplicate_nodes_and_edges(nodes, edges)
return {
"nodes": list(
map(
lambda node: {
"id": str(node.id),
"label": node.name if hasattr(node, "name") else f"{node.type}_{str(node.id)}",
"properties": {},
},
nodes,
)
),
"edges": list(
map(
lambda edge: {
"source": str(edge[0]),
"target": str(edge[1]),
"label": edge[2],
},
edges,
)
),
}

49
cognee/api/v1/datasets/routers/get_datasets_router.py
View file

@ -39,6 +39,23 @@ class DataDTO(OutDTO):
raw_data_location: str
class GraphNodeDTO(OutDTO):
id: UUID
label: str
properties: dict
class GraphEdgeDTO(OutDTO):
source: UUID
target: UUID
label: str
class GraphDTO(OutDTO):
nodes: List[GraphNodeDTO]
edges: List[GraphEdgeDTO]
def get_datasets_router() -> APIRouter:
router = APIRouter()
@ -94,24 +111,46 @@ def get_datasets_router() -> APIRouter:
await delete_data(data)
@router.get("/{dataset_id}/graph", response_model=str)
@router.get("/{dataset_id}/graph", response_model=GraphDTO)
async def get_dataset_graph(dataset_id: UUID, user: User = Depends(get_authenticated_user)):
from cognee.shared.utils import render_graph
from cognee.infrastructure.databases.graph import get_graph_engine
try:
graph_client = await get_graph_engine()
graph_url = await render_graph(graph_client.graph)
(nodes, edges) = await graph_client.get_graph_data()
return JSONResponse(
status_code=200,
content=str(graph_url),
content={
"nodes": list(
map(
lambda node: {
"id": str(node[0]),
"label": node[1]["name"]
if hasattr(node[1], "name")
else f"{node[1]['type']}_{str(node[0])}",
"properties": {},
},
nodes,
)
),
"edges": list(
map(
lambda edge: {
"source": str(edge[0]),
"target": str(edge[1]),
"label": edge[2],
},
edges,
)
),
},
)
except Exception as error:
print(error)
return JSONResponse(
status_code=409,
content="Graphistry credentials are not set. Please set them in your .env file.",
content="Error retrieving dataset graph data.",
)
@router.get(

5
cognee/eval_framework/corpus_builder/corpus_builder_executor.py
View file

@ -61,4 +61,7 @@ class CorpusBuilderExecutor:
await cognee.add(self.raw_corpus)
tasks = await self.task_getter(chunk_size=chunk_size, chunker=chunker)
await cognee_pipeline(tasks=tasks)
pipeline_run = cognee_pipeline(tasks=tasks)
async for run_info in pipeline_run:
print(run_info)

1
cognee/infrastructure/databases/graph/kuzu/adapter.py
View file

@ -1158,6 +1158,7 @@ class KuzuAdapter(GraphDBInterface):
data = json.loads(props)
except json.JSONDecodeError:
logger.warning(f"Failed to parse JSON props for edge {from_id}->{to_id}")
edges.append((from_id, to_id, rel_type, data))
return nodes, edges

57
cognee/modules/metrics/operations/get_pipeline_run_metrics.py
View file

@ -1,10 +1,11 @@
from cognee.infrastructure.databases.relational import get_relational_engine
from sqlalchemy import select
from sqlalchemy.sql import func
from cognee.modules.data.models import Data
from cognee.modules.data.models import GraphMetrics
from cognee.modules.pipelines.models import PipelineRunInfo
from cognee.infrastructure.databases.graph import get_graph_engine
from cognee.modules.pipelines.models import PipelineRun
from cognee.infrastructure.databases.relational import get_relational_engine
async def fetch_token_count(db_engine) -> int:
@ -22,39 +23,39 @@ async def fetch_token_count(db_engine) -> int:
return token_count_sum
async def get_pipeline_run_metrics(pipeline_runs: list[PipelineRun], include_optional: bool):
async def get_pipeline_run_metrics(pipeline_run: PipelineRunInfo, include_optional: bool):
db_engine = get_relational_engine()
graph_engine = await get_graph_engine()
metrics_for_pipeline_runs = []
async with db_engine.get_async_session() as session:
for pipeline_run in pipeline_runs:
existing_metrics = await session.execute(
select(GraphMetrics).where(GraphMetrics.id == pipeline_run.pipeline_run_id)
)
existing_metrics = existing_metrics.scalars().first()
existing_metrics = await session.execute(
select(GraphMetrics).where(GraphMetrics.id == pipeline_run.pipeline_run_id)
)
existing_metrics = existing_metrics.scalars().first()
if existing_metrics:
metrics_for_pipeline_runs.append(existing_metrics)
else:
graph_metrics = await graph_engine.get_graph_metrics(include_optional)
metrics = GraphMetrics(
id=pipeline_run.pipeline_run_id,
num_tokens=await fetch_token_count(db_engine),
num_nodes=graph_metrics["num_nodes"],
num_edges=graph_metrics["num_edges"],
mean_degree=graph_metrics["mean_degree"],
edge_density=graph_metrics["edge_density"],
num_connected_components=graph_metrics["num_connected_components"],
sizes_of_connected_components=graph_metrics["sizes_of_connected_components"],
num_selfloops=graph_metrics["num_selfloops"],
diameter=graph_metrics["diameter"],
avg_shortest_path_length=graph_metrics["avg_shortest_path_length"],
avg_clustering=graph_metrics["avg_clustering"],
)
metrics_for_pipeline_runs.append(metrics)
session.add(metrics)
if existing_metrics:
metrics_for_pipeline_runs.append(existing_metrics)
else:
graph_metrics = await graph_engine.get_graph_metrics(include_optional)
metrics = GraphMetrics(
id=pipeline_run.pipeline_run_id,
num_tokens=await fetch_token_count(db_engine),
num_nodes=graph_metrics["num_nodes"],
num_edges=graph_metrics["num_edges"],
mean_degree=graph_metrics["mean_degree"],
edge_density=graph_metrics["edge_density"],
num_connected_components=graph_metrics["num_connected_components"],
sizes_of_connected_components=graph_metrics["sizes_of_connected_components"],
num_selfloops=graph_metrics["num_selfloops"],
diameter=graph_metrics["diameter"],
avg_shortest_path_length=graph_metrics["avg_shortest_path_length"],
avg_clustering=graph_metrics["avg_clustering"],
)
metrics_for_pipeline_runs.append(metrics)
session.add(metrics)
await session.commit()
return metrics_for_pipeline_runs

33
cognee/modules/pipelines/models/PipelineRunInfo.py Normal file
View file

@ -0,0 +1,33 @@
from typing import Any, Optional
from uuid import UUID
from pydantic import BaseModel
class PipelineRunInfo(BaseModel):
status: str
pipeline_run_id: UUID
payload: Optional[Any] = None
model_config = {
"arbitrary_types_allowed": True,
}
class PipelineRunStarted(PipelineRunInfo):
status: str = "PipelineRunStarted"
pass
class PipelineRunYield(PipelineRunInfo):
status: str = "PipelineRunYield"
pass
class PipelineRunCompleted(PipelineRunInfo):
status: str = "PipelineRunCompleted"
pass
class PipelineRunErrored(PipelineRunInfo):
status: str = "PipelineRunErrored"
pass

7
cognee/modules/pipelines/models/__init__.py
View file

@ -1 +1,8 @@
from .PipelineRun import PipelineRun, PipelineRunStatus
from .PipelineRunInfo import (
PipelineRunInfo,
PipelineRunStarted,
PipelineRunYield,
PipelineRunCompleted,
PipelineRunErrored,
)

5
cognee/modules/pipelines/operations/log_pipeline_run_initiated.py
View file

@ -1,11 +1,12 @@
from uuid import UUID, uuid4
from uuid import UUID
from cognee.infrastructure.databases.relational import get_relational_engine
from cognee.modules.pipelines.models import PipelineRun, PipelineRunStatus
from cognee.modules.pipelines.utils import generate_pipeline_run_id
async def log_pipeline_run_initiated(pipeline_id: str, pipeline_name: str, dataset_id: UUID):
pipeline_run = PipelineRun(
pipeline_run_id=uuid4(),
pipeline_run_id=generate_pipeline_run_id(pipeline_id, dataset_id),
pipeline_name=pipeline_name,
pipeline_id=pipeline_id,
status=PipelineRunStatus.DATASET_PROCESSING_INITIATED,

4
cognee/modules/pipelines/operations/log_pipeline_run_start.py
View file

@ -4,6 +4,8 @@ from cognee.modules.data.models import Data
from cognee.modules.pipelines.models import PipelineRun, PipelineRunStatus
from typing import Any
from cognee.modules.pipelines.utils import generate_pipeline_run_id
async def log_pipeline_run_start(pipeline_id: str, pipeline_name: str, dataset_id: UUID, data: Any):
if not data:
@ -13,7 +15,7 @@ async def log_pipeline_run_start(pipeline_id: str, pipeline_name: str, dataset_i
else:
data_info = str(data)
pipeline_run_id = uuid4()
pipeline_run_id = generate_pipeline_run_id(pipeline_id, dataset_id)
pipeline_run = PipelineRun(
pipeline_run_id=pipeline_run_id,

30
cognee/modules/pipelines/operations/pipeline.py
View file

@ -90,21 +90,16 @@ async def cognee_pipeline(
if not datasets:
raise DatasetNotFoundError("There are no datasets to work with.")
awaitables = []
for dataset in datasets:
awaitables.append(
run_pipeline(
dataset=dataset,
user=user,
tasks=tasks,
data=data,
pipeline_name=pipeline_name,
context={"dataset": dataset},
)
)
return await asyncio.gather(*awaitables)
async for run_info in run_pipeline(
dataset=dataset,
user=user,
tasks=tasks,
data=data,
pipeline_name=pipeline_name,
context={"dataset": dataset},
):
yield run_info
async def run_pipeline(
@ -169,9 +164,6 @@ async def run_pipeline(
raise ValueError(f"Task {task} is not an instance of Task")
pipeline_run = run_tasks(tasks, dataset_id, data, user, pipeline_name, context=context)
pipeline_run_status = None
async for run_status in pipeline_run:
pipeline_run_status = run_status
return pipeline_run_status
async for pipeline_run_info in pipeline_run:
yield pipeline_run_info

48
cognee/modules/pipelines/operations/run_tasks.py
View file

@ -1,18 +1,25 @@
import json
from cognee.shared.logging_utils import get_logger
from typing import Any
from uuid import UUID, uuid4
from typing import Any
from cognee.shared.logging_utils import get_logger
from cognee.modules.users.methods import get_default_user
from cognee.modules.pipelines.utils import generate_pipeline_id
from cognee.modules.pipelines.models.PipelineRunInfo import (
PipelineRunCompleted,
PipelineRunErrored,
PipelineRunStarted,
PipelineRunYield,
)
from cognee.modules.pipelines.operations import (
log_pipeline_run_start,
log_pipeline_run_complete,
log_pipeline_run_error,
)
from cognee.modules.settings import get_current_settings
from cognee.modules.users.methods import get_default_user
from cognee.modules.users.models import User
from cognee.shared.utils import send_telemetry
from uuid import uuid5, NAMESPACE_OID
from .run_tasks_base import run_tasks_base
from ..tasks.task import Task
@ -76,29 +83,44 @@ async def run_tasks(
pipeline_name: str = "unknown_pipeline",
context: dict = None,
):
pipeline_id = uuid5(NAMESPACE_OID, pipeline_name)
if not user:
user = get_default_user()
pipeline_id = generate_pipeline_id(user.id, pipeline_name)
pipeline_run = await log_pipeline_run_start(pipeline_id, pipeline_name, dataset_id, data)
yield pipeline_run
pipeline_run_id = pipeline_run.pipeline_run_id
yield PipelineRunStarted(
pipeline_run_id=pipeline_run_id,
payload=data,
)
try:
async for _ in run_tasks_with_telemetry(
async for result in run_tasks_with_telemetry(
tasks=tasks,
data=data,
user=user,
pipeline_name=pipeline_id,
context=context,
):
pass
yield PipelineRunYield(
pipeline_run_id=pipeline_run_id,
payload=result,
)
yield await log_pipeline_run_complete(
await log_pipeline_run_complete(
pipeline_run_id, pipeline_id, pipeline_name, dataset_id, data
)
except Exception as e:
yield await log_pipeline_run_error(
pipeline_run_id, pipeline_id, pipeline_name, dataset_id, data, e
yield PipelineRunCompleted(pipeline_run_id=pipeline_run_id)
except Exception as error:
await log_pipeline_run_error(
pipeline_run_id, pipeline_id, pipeline_name, dataset_id, data, error
)
raise e
yield PipelineRunErrored(payload=error)
raise error

37
cognee/modules/pipelines/queues/pipeline_run_info_queues.py Normal file
View file

@ -0,0 +1,37 @@
from uuid import UUID
from asyncio import Queue
from typing import Optional
from cognee.modules.pipelines.models import PipelineRunInfo
pipeline_run_info_queues = {}
def initialize_queue(pipeline_run_id: UUID):
pipeline_run_info_queues[str(pipeline_run_id)] = Queue()
def get_queue(pipeline_run_id: UUID) -> Optional[Queue]:
if str(pipeline_run_id) not in pipeline_run_info_queues:
initialize_queue(pipeline_run_id)
return pipeline_run_info_queues.get(str(pipeline_run_id), None)
def remove_queue(pipeline_run_id: UUID):
pipeline_run_info_queues.pop(str(pipeline_run_id))
def push_to_queue(pipeline_run_id: UUID, pipeline_run_info: PipelineRunInfo):
queue = get_queue(pipeline_run_id)
if queue:
queue.put_nowait(pipeline_run_info)
def get_from_queue(pipeline_run_id: UUID) -> Optional[PipelineRunInfo]:
queue = get_queue(pipeline_run_id)
item = queue.get_nowait() if queue and not queue.empty() else None
return item

2
cognee/modules/pipelines/utils/__init__.py Normal file
View file

@ -0,0 +1,2 @@
from .generate_pipeline_id import generate_pipeline_id
from .generate_pipeline_run_id import generate_pipeline_run_id

5
cognee/modules/pipelines/utils/generate_pipeline_id.py Normal file
View file

@ -0,0 +1,5 @@
from uuid import NAMESPACE_OID, UUID, uuid5
def generate_pipeline_id(user_id: UUID, pipeline_name: str):
return uuid5(NAMESPACE_OID, f"{str(user_id)}_{pipeline_name}")

5
cognee/modules/pipelines/utils/generate_pipeline_run_id.py Normal file
View file

@ -0,0 +1,5 @@
from uuid import NAMESPACE_OID, UUID, uuid5
def generate_pipeline_run_id(pipeline_id: UUID, dataset_id: UUID):
return uuid5(NAMESPACE_OID, f"{str(pipeline_id)}_{str(dataset_id)}")

10
cognee/modules/retrieval/exceptions/exceptions.py
View file

@ -24,3 +24,13 @@ class CypherSearchError(CogneeApiError):
class NoDataError(CriticalError):
message: str = "No data found in the system, please add data first."
class CollectionDistancesNotFoundError(CogneeApiError):
def __init__(
self,
message: str = "No collection distances found for the given query.",
name: str = "CollectionDistancesNotFoundError",
status_code: int = status.HTTP_404_NOT_FOUND,
):
super().__init__(message, name, status_code)

4
cognee/modules/retrieval/graph_completion_retriever.py
View file

@ -8,6 +8,9 @@ from cognee.modules.retrieval.base_retriever import BaseRetriever
from cognee.modules.retrieval.utils.brute_force_triplet_search import brute_force_triplet_search
from cognee.modules.retrieval.utils.completion import generate_completion
from cognee.modules.retrieval.utils.stop_words import DEFAULT_STOP_WORDS
from cognee.shared.logging_utils import get_logger
logger = get_logger()
class GraphCompletionRetriever(BaseRetriever):
@ -133,6 +136,7 @@ class GraphCompletionRetriever(BaseRetriever):
triplets = await self.get_triplets(query)
if len(triplets) == 0:
logger.warning("Empty context was provided to the completion")
return ""
return await self.resolve_edges_to_text(triplets)

2
cognee/modules/retrieval/utils/brute_force_triplet_search.py
View file

@ -125,6 +125,8 @@ async def brute_force_search(
collections (Optional[List[str]]): List of collections to query.
properties_to_project (Optional[List[str]]): List of properties to project.
memory_fragment (Optional[CogneeGraph]): Existing memory fragment to reuse.
node_type: node type to filter
node_name: node name to filter
Returns:
list: The top triplet results.

2
cognee/tests/test_cognee_server_start.py
View file

@ -27,7 +27,7 @@ class TestCogneeServerStart(unittest.TestCase):
preexec_fn=os.setsid,
)
# Give the server some time to start
time.sleep(20)
time.sleep(30)
# Check if server started with errors
if cls.server_process.poll() is not None:

34
poetry.lock generated
View file

@ -1,4 +1,4 @@
# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand.
[[package]]
name = "aiobotocore"
@ -454,7 +454,7 @@ description = "Timeout context manager for asyncio programs"
optional = true
python-versions = ">=3.8"
groups = ["main"]
markers = "python_version == \"3.11\" and python_full_version < \"3.11.3\" and extra == \"falkordb\""
markers = "extra == \"falkordb\" and python_full_version < \"3.11.3\" and python_version == \"3.11\""
files = [
{file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"},
{file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"},
@ -599,7 +599,7 @@ description = "Backport of CPython tarfile module"
optional = true
python-versions = ">=3.8"
groups = ["main"]
markers = "(python_version == \"3.10\" or python_version == \"3.11\") and extra == \"deepeval\""
markers = "extra == \"deepeval\" and python_version < \"3.12\""
files = [
{file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"},
{file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"},
@ -1232,7 +1232,7 @@ description = "Cross-platform colored terminal text."
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
groups = ["main"]
markers = "(sys_platform == \"win32\" or platform_system == \"Windows\" or extra == \"llama-index\" or extra == \"deepeval\" or extra == \"dev\") and (sys_platform == \"win32\" or platform_system == \"Windows\" or extra == \"llama-index\" or extra == \"deepeval\" or extra == \"dev\" or extra == \"chromadb\") and (sys_platform == \"win32\" or platform_system == \"Windows\" or extra == \"llama-index\" or extra == \"deepeval\" or extra == \"dev\" or extra == \"chromadb\" or extra == \"codegraph\") and (sys_platform == \"win32\" or platform_system == \"Windows\" or extra == \"notebook\" or extra == \"dev\" or extra == \"llama-index\" or extra == \"deepeval\") and (sys_platform == \"win32\" or platform_system == \"Windows\" or extra == \"notebook\" or extra == \"dev\" or extra == \"llama-index\" or extra == \"deepeval\" or extra == \"chromadb\") and (platform_system == \"Windows\" or extra == \"notebook\" or extra == \"dev\" or extra == \"llama-index\" or extra == \"deepeval\" or extra == \"chromadb\" or extra == \"codegraph\") and (python_version < \"3.13\" or platform_system == \"Windows\" or extra == \"notebook\" or extra == \"dev\" or extra == \"llama-index\" or extra == \"deepeval\" or extra == \"chromadb\") and (python_version == \"3.10\" or python_version == \"3.11\" or python_version == \"3.12\" or platform_system == \"Windows\" or extra == \"notebook\" or extra == \"dev\" or extra == \"llama-index\" or extra == \"deepeval\" or extra == \"chromadb\")"
markers = "(platform_system == \"Windows\" or sys_platform == \"win32\" or extra == \"llama-index\" or extra == \"deepeval\" or extra == \"dev\") and (python_version <= \"3.12\" or platform_system == \"Windows\" or extra == \"notebook\" or extra == \"dev\" or extra == \"llama-index\" or extra == \"deepeval\" or extra == \"chromadb\") and (platform_system == \"Windows\" or extra == \"notebook\" or extra == \"dev\" or extra == \"llama-index\" or extra == \"deepeval\" or extra == \"chromadb\" or extra == \"codegraph\")"
files = [
{file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
@ -2141,7 +2141,7 @@ description = "Fast, light, accurate library built for retrieval embedding gener
optional = true
python-versions = ">=3.9.0"
groups = ["main"]
markers = "python_version == \"3.10\" and extra == \"codegraph\" or python_version == \"3.11\" and extra == \"codegraph\" or python_version == \"3.12\" and extra == \"codegraph\""
markers = "extra == \"codegraph\" and python_version <= \"3.12\""
files = [
{file = "fastembed-0.6.0-py3-none-any.whl", hash = "sha256:a08385e9388adea0529a586004f2d588c9787880a510e4e5d167127a11e75328"},
{file = "fastembed-0.6.0.tar.gz", hash = "sha256:5c9ead25f23449535b07243bbe1f370b820dcc77ec2931e61674e3fe7ff24733"},
@ -2881,7 +2881,7 @@ description = "HTTP/2-based RPC framework"
optional = true
python-versions = ">=3.8"
groups = ["main"]
markers = "extra == \"gemini\" or extra == \"deepeval\" or extra == \"weaviate\" or extra == \"qdrant\" or extra == \"milvus\" or python_version == \"3.10\" and (extra == \"deepeval\" or extra == \"weaviate\" or extra == \"qdrant\" or extra == \"gemini\" or extra == \"milvus\")"
markers = "extra == \"gemini\" or extra == \"weaviate\" or extra == \"qdrant\" or extra == \"deepeval\" or extra == \"milvus\" or python_version == \"3.10\" and (extra == \"deepeval\" or extra == \"weaviate\" or extra == \"qdrant\" or extra == \"gemini\" or extra == \"milvus\")"
files = [
{file = "grpcio-1.67.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:8b0341d66a57f8a3119b77ab32207072be60c9bf79760fa609c5609f2deb1f3f"},
{file = "grpcio-1.67.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:f5a27dddefe0e2357d3e617b9079b4bfdc91341a91565111a21ed6ebbc51b22d"},
@ -5006,7 +5006,7 @@ description = "Python logging made (stupidly) simple"
optional = true
python-versions = "<4.0,>=3.5"
groups = ["main"]
markers = "python_version == \"3.10\" and extra == \"codegraph\" or python_version == \"3.11\" and extra == \"codegraph\" or python_version == \"3.12\" and extra == \"codegraph\""
markers = "extra == \"codegraph\" and python_version <= \"3.12\""
files = [
{file = "loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c"},
{file = "loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6"},
@ -5747,7 +5747,7 @@ description = "Python extension for MurmurHash (MurmurHash3), a set of fast and
optional = true
python-versions = ">=3.9"
groups = ["main"]
markers = "python_version == \"3.10\" and extra == \"codegraph\" or python_version == \"3.11\" and extra == \"codegraph\" or python_version == \"3.12\" and extra == \"codegraph\""
markers = "extra == \"codegraph\" and python_version <= \"3.12\""
files = [
{file = "mmh3-5.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:eaf4ac5c6ee18ca9232238364d7f2a213278ae5ca97897cafaa123fcc7bb8bec"},
{file = "mmh3-5.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:48f9aa8ccb9ad1d577a16104834ac44ff640d8de8c0caed09a2300df7ce8460a"},
@ -6243,7 +6243,7 @@ description = "Python package for creating and manipulating graphs and networks"
optional = false
python-versions = ">=3.11"
groups = ["main"]
markers = "python_version >= \"3.11\""
markers = "python_version >= \"3.11\" and python_version < \"3.13\" or python_full_version == \"3.13.0\""
files = [
{file = "networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec"},
{file = "networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037"},
@ -6382,7 +6382,7 @@ description = "Fundamental package for array computing in Python"
optional = false
python-versions = ">=3.9"
groups = ["main"]
markers = "python_version == \"3.10\" or python_version == \"3.11\""
markers = "python_version < \"3.12\""
files = [
{file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"},
{file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"},
@ -6429,7 +6429,7 @@ description = "Fundamental package for array computing in Python"
optional = false
python-versions = ">=3.10"
groups = ["main"]
markers = "python_version >= \"3.12\""
markers = "python_version == \"3.12\" or python_full_version == \"3.13.0\""
files = [
{file = "numpy-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6326ab99b52fafdcdeccf602d6286191a79fe2fda0ae90573c5814cd2b0bc1b8"},
{file = "numpy-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0937e54c09f7a9a68da6889362ddd2ff584c02d015ec92672c099b61555f8911"},
@ -7714,7 +7714,7 @@ description = "Fast and parallel snowball stemmer"
optional = true
python-versions = "*"
groups = ["main"]
markers = "python_version == \"3.10\" and extra == \"codegraph\" or python_version == \"3.11\" and extra == \"codegraph\" or python_version == \"3.12\" and extra == \"codegraph\""
markers = "extra == \"codegraph\" and python_version <= \"3.12\""
files = [
{file = "py_rust_stemmers-0.1.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:bfbd9034ae00419ff2154e33b8f5b4c4d99d1f9271f31ed059e5c7e9fa005844"},
{file = "py_rust_stemmers-0.1.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7162ae66df2bb0fc39b350c24a049f5f5151c03c046092ba095c2141ec223a2"},
@ -8623,7 +8623,7 @@ description = "Python for Window Extensions"
optional = false
python-versions = "*"
groups = ["main"]
markers = "(extra == \"qdrant\" or extra == \"deepeval\") and (extra == \"qdrant\" or extra == \"deepeval\" or extra == \"notebook\" or extra == \"dev\") and platform_system == \"Windows\" or sys_platform == \"win32\""
markers = "(sys_platform == \"win32\" or extra == \"qdrant\" or extra == \"deepeval\") and (sys_platform == \"win32\" or platform_system == \"Windows\")"
files = [
{file = "pywin32-310-cp310-cp310-win32.whl", hash = "sha256:6dd97011efc8bf51d6793a82292419eba2c71cf8e7250cfac03bba284454abc1"},
{file = "pywin32-310-cp310-cp310-win_amd64.whl", hash = "sha256:c3e78706e4229b915a0821941a84e7ef420bf2b77e08c9dae3c76fd03fd2ae3d"},
@ -11410,7 +11410,7 @@ description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
optional = true
python-versions = ">=3.9"
groups = ["main"]
markers = "extra == \"chromadb\""
markers = "extra == \"api\" or extra == \"chromadb\""
files = [
{file = "websockets-15.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d63efaa0cd96cf0c5fe4d581521d9fa87744540d4bc999ae6e08595a1014b45b"},
{file = "websockets-15.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac60e3b188ec7574cb761b08d50fcedf9d77f1530352db4eef1707fe9dee7205"},
@ -11530,7 +11530,7 @@ description = "A small Python utility to set file creation time on Windows"
optional = true
python-versions = ">=3.5"
groups = ["main"]
markers = "(python_version == \"3.10\" or python_version == \"3.11\" or python_version == \"3.12\") and extra == \"codegraph\" and sys_platform == \"win32\" and python_version < \"3.13\""
markers = "sys_platform == \"win32\" and extra == \"codegraph\" and python_version <= \"3.12\""
files = [
{file = "win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390"},
{file = "win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0"},
@ -11914,7 +11914,7 @@ cffi = ["cffi (>=1.11)"]
[extras]
anthropic = ["anthropic"]
api = ["gunicorn", "kuzu", "uvicorn"]
api = ["gunicorn", "kuzu", "uvicorn", "websockets"]
chromadb = ["chromadb", "pypika"]
codegraph = ["fastembed", "transformers", "tree-sitter", "tree-sitter-python"]
debug = ["debugpy"]
@ -11944,4 +11944,4 @@ weaviate = ["weaviate-client"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<=3.13"
content-hash = "be9a1d758f9a2dbc0f143b85b82779112c25e3166b213f8076504c87b0242891"
content-hash = "933deb7b39e77b71d479b02cd1ba70f7bc8a75b3c96b186d9208a26f90742906"

1
pyproject.toml
View file

@ -64,6 +64,7 @@ dependencies = [
api = [
"uvicorn==0.34.0",
"gunicorn>=20.1.0,<21",
"websockets>=15.0.1",
"kuzu==0.9.0",
]
weaviate = ["weaviate-client==4.9.6"]

2
uv.lock generated
View file

@ -920,6 +920,7 @@ api = [
{ name = "gunicorn" },
{ name = "kuzu" },
{ name = "uvicorn" },
{ name = "websockets" },
]
chromadb = [
{ name = "chromadb" },
@ -1109,6 +1110,7 @@ requires-dist = [
{ name = "unstructured", extras = ["csv", "doc", "docx", "epub", "md", "odt", "org", "ppt", "pptx", "rst", "rtf", "tsv", "xlsx"], marker = "extra == 'docs'", specifier = ">=0.16.13,<18" },
{ name = "uvicorn", marker = "extra == 'api'", specifier = "==0.34.0" },
{ name = "weaviate-client", marker = "extra == 'weaviate'", specifier = "==4.9.6" },
{ name = "websockets", marker = "extra == 'api'", specifier = ">=15.0.1" },
]
provides-extras = ["api", "weaviate", "qdrant", "neo4j", "postgres", "notebook", "langchain", "llama-index", "gemini", "huggingface", "ollama", "mistral", "anthropic", "deepeval", "posthog", "falkordb", "kuzu", "groq", "milvus", "chromadb", "docs", "codegraph", "evals", "gui", "graphiti", "dev", "debug"]