gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: use cloud in local (#1367)

Browse source 
<!-- .github/pull_request_template.md -->

## Description
<!-- Provide a clear description of the changes in this PR -->

## DCO Affirmation
I affirm that all code in every commit of this pull request conforms to
the terms of the Topoteretes Developer Certificate of Origin.
This commit is contained in:
Boris 2025-09-10 16:33:32 +02:00 • committed by GitHub
parent b1643414d2
commit 68ec07c78a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 159 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
cognee-frontend/src/app/(graph)/CogneeAddWidget.tsx
View file

@ -26,9 +26,10 @@ export interface NodesAndEdges {
interface CogneeAddWidgetProps {
onData: (data: NodesAndLinks) => void;
useCloud?: boolean;
}
export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
export default function CogneeAddWidget({ onData, useCloud = false }: CogneeAddWidgetProps) {
const {
datasets,
refreshDatasets,
@ -76,17 +77,18 @@ export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
return addData(dataset, files)
.then(() => {
const onUpdate = (data: NodesAndEdges) => {
onData({
nodes: data.nodes,
links: data.edges,
});
setProcessingFilesDone();
};
// const onUpdate = (data: NodesAndEdges) => {
// onData({
// nodes: data.nodes,
// links: data.edges,
// });
// setProcessingFilesDone();
// };
return cognifyDataset(dataset, onUpdate)
return cognifyDataset(dataset, useCloud)
.then(() => {
refreshDatasets();
setProcessingFilesDone();
});
});
};

7
cognee-frontend/src/app/dashboard/AddDataToCognee.tsx
View file

@ -10,9 +10,10 @@ import cognifyDataset from "@/modules/datasets/cognifyDataset";
interface AddDataToCogneeProps {
datasets: Dataset[];
refreshDatasets: () => void;
useCloud?: boolean;
}
export default function AddDataToCognee({ datasets, refreshDatasets }: AddDataToCogneeProps) {
export default function AddDataToCognee({ datasets, refreshDatasets, useCloud = false }: AddDataToCogneeProps) {
const [filesForUpload, setFilesForUpload] = useState<FileList | null>(null);
const prepareFiles = useCallback((event: FormEvent<HTMLInputElement>) => {
@ -49,9 +50,9 @@ export default function AddDataToCognee({ datasets, refreshDatasets }: AddDataTo
name: dataset_name,
data: [], // not important, just to mimick Dataset
status: "", // not important, just to mimick Dataset
});
}, useCloud);
});
}, [filesForUpload, refreshDatasets]);
}, [filesForUpload, refreshDatasets, useCloud]);
const {
isModalOpen: isAddDataModalOpen,

9
cognee-frontend/src/app/dashboard/DatasetsAccordion.tsx
View file

@ -18,6 +18,7 @@ interface DatasetsChangePayload {
export interface DatasetsAccordionProps extends Omit<AccordionProps, "isOpen" | "openAccordion" | "closeAccordion" | "children"> {
onDatasetsChange?: (payload: DatasetsChangePayload) => void;
useCloud?: boolean;
}
export default function DatasetsAccordion({
@ -27,6 +28,7 @@ export default function DatasetsAccordion({
className,
contentClassName,
onDatasetsChange,
useCloud = false,
}: DatasetsAccordionProps) {
const {
value: isDatasetsPanelOpen,
@ -41,7 +43,7 @@ export default function DatasetsAccordion({
removeDataset,
getDatasetData,
removeDatasetData,
} = useDatasets();
} = useDatasets(useCloud);
useEffect(() => {
if (datasets.length === 0) {
@ -177,11 +179,11 @@ export default function DatasetsAccordion({
return;
}
return addData(dataset, files)
return addData(dataset, files, useCloud)
.then(async () => {
await getDatasetData(dataset.id);
return cognifyDataset(dataset)
return cognifyDataset(dataset, useCloud)
.finally(() => {
setProcessingDataset(null);
});
@ -255,7 +257,6 @@ export default function DatasetsAccordion({
closeAccordion={() => toggleDataset(dataset.id)}
tools={(
<IconButton className="relative">
<input tabIndex={-1} type="file" multiple onChange={handleAddFiles.bind(null, dataset)} className="absolute w-full h-full cursor-pointer opacity-0" />
<PopupMenu>
<div className="flex flex-col gap-0.5">
<div className="hover:bg-gray-100 w-full text-left px-2 cursor-pointer relative">

39
cognee-frontend/src/app/dashboard/InstanceDatasetsAccordion.tsx
View file

@ -2,7 +2,7 @@ import { useCallback, useEffect } from "react";
import { fetch, useBoolean } from "@/utils";
import { checkCloudConnection } from "@/modules/cloud";
import { CloseIcon, CloudIcon, LocalCogneeIcon } from "@/ui/Icons";
import { CaretIcon, CloseIcon, CloudIcon, LocalCogneeIcon } from "@/ui/Icons";
import { CTAButton, GhostButton, IconButton, Input, Modal } from "@/ui/elements";
import DatasetsAccordion, { DatasetsAccordionProps } from "./DatasetsAccordion";
@ -21,6 +21,7 @@ export default function InstanceDatasetsAccordion({ onDatasetsChange }: Instance
} = useBoolean(false);
const checkConnectionToCloudCognee = useCallback((apiKey: string) => {
fetch.setApiKey(apiKey);
return checkCloudConnection(apiKey)
.then(setCloudCogneeConnected)
}, [setCloudCogneeConnected]);
@ -71,13 +72,35 @@ export default function InstanceDatasetsAccordion({ onDatasetsChange }: Instance
onDatasetsChange={onDatasetsChange}
/>
<button className="w-full flex flex-row items-center justify-between py-1.5 cursor-pointer" onClick={!isCloudCogneeConnected ? openCloudConnectionModal : () => {}}>
<div className="flex flex-row items-center gap-2">
<CloudIcon color="#000000" />
<span className="text-xs">cloud cognee</span>
</div>
{isCloudCogneeConnected ? <span className="text-xs text-indigo-600">Connected</span> : <span className="text-xs text-gray-400">Not connected</span>}
</button>
{isCloudCogneeConnected ? (
<DatasetsAccordion
title={(
<div className="flex flex-row items-center justify-between">
<div className="flex flex-row items-center gap-2">
<LocalCogneeIcon className="text-indigo-700" />
<span className="text-xs">cloud cognee</span>
</div>
</div>
)}
tools={<span className="text-xs text-indigo-600">Connected</span>}
switchCaretPosition={true}
className="pt-3 pb-1.5"
contentClassName="pl-4"
onDatasetsChange={onDatasetsChange}
useCloud={true}
/>
) : (
<button className="w-full flex flex-row items-center justify-between py-1.5 cursor-pointer" onClick={!isCloudCogneeConnected ? openCloudConnectionModal : () => {}}>
<div className="flex flex-row items-center gap-1.5">
<CaretIcon className="rotate-[-90deg]" />
<div className="flex flex-row items-center gap-2">
<CloudIcon color="#000000" />
<span className="text-xs">cloud cognee</span>
</div>
</div>
<span className="text-xs text-gray-400">Not connected</span>
</button>
)}
<Modal isOpen={isCloudConnectedModalOpen}>
<div className="w-full max-w-2xl">

4
cognee-frontend/src/modules/datasets/cognifyDataset.ts
View file

@ -7,7 +7,7 @@ import { Dataset } from "../ingestion/useDatasets";
// edges: { source: string; target: string; label: string }[];
// }
export default async function cognifyDataset(dataset: Dataset) {
export default async function cognifyDataset(dataset: Dataset, useCloud: boolean = false) {
// const data = await (
return fetch("/v1/cognify", {
method: "POST",
@ -18,7 +18,7 @@ export default async function cognifyDataset(dataset: Dataset) {
datasetIds: [dataset.id],
runInBackground: false,
}),
})
}, useCloud)
.then((response) => response.json());
// .then(() => {
// return getDatasetGraph(dataset)

6
cognee-frontend/src/modules/datasets/createDataset.ts
View file

@ -1,12 +1,12 @@
import { fetch } from "@/utils";
export default function createDataset(dataset: { name: string }) {
export default function createDataset(dataset: { name: string }, useCloud = false) {
return fetch(`/v1/datasets/`, {
method: "POST",
body: JSON.stringify(dataset),
headers: {
"Content-Type": "application/json",
}
})
},
}, useCloud)
.then((response) => response.json());
}

46
cognee-frontend/src/modules/ingestion/addData.ts
View file

@ -1,19 +1,35 @@
import { fetch } from "@/utils";
export default function addData(dataset: { id?: string, name?: string }, files: File[]) {
const formData = new FormData();
files.forEach((file) => {
formData.append("data", file, file.name);
})
if (dataset.id) {
formData.append("datasetId", dataset.id);
}
if (dataset.name) {
formData.append("datasetName", dataset.name);
}
export default async function addData(dataset: { id?: string, name?: string }, files: File[], useCloud = false) {
if (useCloud) {
const data = {
text_data: await Promise.all(files.map(async (file) => file.text())),
datasetId: dataset.id,
datasetName: dataset.name,
};
return fetch("/v1/add", {
method: "POST",
body: formData,
}).then((response) => response.json());
return fetch("/v1/add", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(data),
}, true).then((response) => response.json());
} else {
const formData = new FormData();
files.forEach((file) => {
formData.append("data", file, file.name);
})
if (dataset.id) {
formData.append("datasetId", dataset.id);
}
if (dataset.name) {
formData.append("datasetName", dataset.name);
}
return fetch("/v1/add", {
method: "POST",
body: formData,
}).then((response) => response.json());
}
}

104
cognee-frontend/src/modules/ingestion/useDatasets.ts
View file

@ -1,5 +1,4 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import { v4 } from 'uuid';
import { useCallback, useState } from 'react';
import { fetch } from '@/utils';
import { DataFile } from './useData';
@ -12,95 +11,96 @@ export interface Dataset {
status: string;
}
function useDatasets() {
function useDatasets(useCloud = false) {
const [datasets, setDatasets] = useState<Dataset[]>([]);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const statusTimeout = useRef<any>(null);
// const statusTimeout = useRef<any>(null);
const fetchDatasetStatuses = useCallback((datasets: Dataset[]) => {
fetch(
`/v1/datasets/status?dataset=${datasets.map(d => d.id).join('&dataset=')}`,
{
headers: {
"Content-Type": "application/json",
},
},
)
.then((response) => response.json())
.then((statuses) => setDatasets(
(datasets) => (
datasets.map((dataset) => ({
...dataset,
status: statuses[dataset.id]
}))
)));
}, []);
// const fetchDatasetStatuses = useCallback((datasets: Dataset[]) => {
// fetch(
// `/v1/datasets/status?dataset=${datasets.map(d => d.id).join('&dataset=')}`,
// {
// headers: {
// "Content-Type": "application/json",
// },
// },
// useCloud,
// )
// .then((response) => response.json())
// .then((statuses) => setDatasets(
// (datasets) => (
// datasets.map((dataset) => ({
// ...dataset,
// status: statuses[dataset.id]
// }))
// )));
// }, [useCloud]);
const checkDatasetStatuses = useCallback((datasets: Dataset[]) => {
fetchDatasetStatuses(datasets);
// const checkDatasetStatuses = useCallback((datasets: Dataset[]) => {
// fetchDatasetStatuses(datasets);
if (statusTimeout.current !== null) {
clearTimeout(statusTimeout.current);
}
// if (statusTimeout.current !== null) {
// clearTimeout(statusTimeout.current);
// }
statusTimeout.current = setTimeout(() => {
checkDatasetStatuses(datasets);
}, 50000);
}, [fetchDatasetStatuses]);
// statusTimeout.current = setTimeout(() => {
// checkDatasetStatuses(datasets);
// }, 50000);
// }, [fetchDatasetStatuses]);
useEffect(() => {
return () => {
if (statusTimeout.current !== null) {
clearTimeout(statusTimeout.current);
statusTimeout.current = null;
}
};
}, []);
// useEffect(() => {
// return () => {
// if (statusTimeout.current !== null) {
// clearTimeout(statusTimeout.current);
// statusTimeout.current = null;
// }
// };
// }, []);
const addDataset = useCallback((datasetName: string) => {
return createDataset({ name: datasetName })
return createDataset({ name: datasetName }, useCloud)
.then((dataset) => {
setDatasets((datasets) => [
...datasets,
dataset,
]);
});
}, []);
}, [useCloud]);
const removeDataset = useCallback((datasetId: string) => {
return fetch(`/v1/datasets/${datasetId}`, {
method: 'DELETE',
})
}, useCloud)
.then(() => {
setDatasets((datasets) =>
datasets.filter((dataset) => dataset.id !== datasetId)
);
});
}, []);
}, [useCloud]);
const fetchDatasets = useCallback(() => {
return fetch('/v1/datasets', {
headers: {
"Content-Type": "application/json",
},
})
}, useCloud)
.then((response) => response.json())
.then((datasets) => {
setDatasets(datasets);
if (datasets.length > 0) {
checkDatasetStatuses(datasets);
}
// if (datasets.length > 0) {
// checkDatasetStatuses(datasets);
// }
return datasets;
})
.catch((error) => {
console.error('Error fetching datasets:', error);
});
}, [checkDatasetStatuses]);
}, [useCloud]);
const getDatasetData = useCallback((datasetId: string) => {
return fetch(`/v1/datasets/${datasetId}/data`)
return fetch(`/v1/datasets/${datasetId}/data`, {}, useCloud)
.then((response) => response.json())
.then((data) => {
const datasetIndex = datasets.findIndex((dataset) => dataset.id === datasetId);
@ -118,13 +118,13 @@ function useDatasets() {
return data;
});
}, [datasets]);
}, [datasets, useCloud]);
const removeDatasetData = useCallback((datasetId: string, dataId: string) => {
return fetch(`/v1/datasets/${datasetId}/data/${dataId}`, {
method: 'DELETE',
});
}, []);
}, useCloud);
}, [useCloud]);
return {
datasets,

25
cognee-frontend/src/utils/fetch.ts
View file

@ -6,7 +6,11 @@ const isAuth0Enabled = process.env.USE_AUTH0_AUTHORIZATION?.toLowerCase() === "t
const backendApiUrl = process.env.NEXT_PUBLIC_BACKEND_API_URL || "http://localhost:8000/api";
export default async function fetch(url: string, options: RequestInit = {}): Promise<Response> {
const cloudApiUrl = process.env.NEXT_PUBLIC_CLOUD_API_URL || "https://api.cognee.ai/api";
let apiKey: string | null = null;
export default async function fetch(url: string, options: RequestInit = {}, useCloud = false): Promise<Response> {
function retry(lastError: Response) {
if (!isAuth0Enabled) {
return Promise.reject(lastError);
@ -24,10 +28,17 @@ export default async function fetch(url: string, options: RequestInit = {}): Pro
});
}
return global.fetch(backendApiUrl + url, {
...options,
credentials: "include",
})
return global.fetch(
(useCloud ? cloudApiUrl : backendApiUrl) + (useCloud ? url.replace("/v1", "") : url),
{
...options,
headers: {
...options.headers,
...(useCloud ? {"X-Api-Key": apiKey!} : {}),
},
credentials: "include",
},
)
.then((response) => handleServerErrors(response, retry))
.then((response) => {
numberOfRetries = 0;
@ -51,3 +62,7 @@ export default async function fetch(url: string, options: RequestInit = {}): Pro
fetch.checkHealth = () => {
return global.fetch(`${backendApiUrl.replace("/api", "")}/health`);
};
fetch.setApiKey = (newApiKey: string) => {
apiKey = newApiKey;
};