feat: use cloud in local (#1367)

<!-- .github/pull_request_template.md -->

## Description
<!-- Provide a clear description of the changes in this PR -->

## DCO Affirmation
I affirm that all code in every commit of this pull request conforms to
the terms of the Topoteretes Developer Certificate of Origin.

cognee-frontend/src/app/(graph)/CogneeAddWidget.tsx

@@ -26,9 +26,10 @@ export interface NodesAndEdges {
 
 interface CogneeAddWidgetProps {
   onData: (data: NodesAndLinks) => void;
+  useCloud?: boolean;
 }
 
-export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
+export default function CogneeAddWidget({ onData, useCloud = false }: CogneeAddWidgetProps) {
   const {
     datasets,
     refreshDatasets,
@@ -76,17 +77,18 @@ export default function CogneeAddWidget({ onData }: CogneeAddWidgetProps) {
 
     return addData(dataset, files)
       .then(() => {
-        const onUpdate = (data: NodesAndEdges) => {
+        // const onUpdate = (data: NodesAndEdges) => {
-          onData({
+        //   onData({
-            nodes: data.nodes,
+        //     nodes: data.nodes,
-            links: data.edges,
+        //     links: data.edges,
-          });
+        //   });
-          setProcessingFilesDone();
+        //   setProcessingFilesDone();
-        };
+        // };
 
-        return cognifyDataset(dataset, onUpdate)
+        return cognifyDataset(dataset, useCloud)
           .then(() => {
             refreshDatasets();
+            setProcessingFilesDone();
           });
       });
   };

cognee-frontend/src/app/dashboard/AddDataToCognee.tsx

@@ -10,9 +10,10 @@ import cognifyDataset from "@/modules/datasets/cognifyDataset";
 interface AddDataToCogneeProps {
   datasets: Dataset[];
   refreshDatasets: () => void;
+  useCloud?: boolean;
 }
 
-export default function AddDataToCognee({ datasets, refreshDatasets }: AddDataToCogneeProps) {
+export default function AddDataToCognee({ datasets, refreshDatasets, useCloud = false }: AddDataToCogneeProps) {
   const [filesForUpload, setFilesForUpload] = useState<FileList | null>(null);
 
   const prepareFiles = useCallback((event: FormEvent<HTMLInputElement>) => {
@@ -49,9 +50,9 @@ export default function AddDataToCognee({ datasets, refreshDatasets }: AddDataTo
           name: dataset_name,
           data: [],  // not important, just to mimick Dataset
           status: "",  // not important, just to mimick Dataset
-        });
+        }, useCloud);
       });
-  }, [filesForUpload, refreshDatasets]);
+  }, [filesForUpload, refreshDatasets, useCloud]);
 
   const {
     isModalOpen: isAddDataModalOpen,

cognee-frontend/src/app/dashboard/DatasetsAccordion.tsx

@@ -18,6 +18,7 @@ interface DatasetsChangePayload {
 
 export interface DatasetsAccordionProps extends Omit<AccordionProps, "isOpen" | "openAccordion" | "closeAccordion" | "children"> {
   onDatasetsChange?: (payload: DatasetsChangePayload) => void;
+  useCloud?: boolean;
 }
 
 export default function DatasetsAccordion({
@@ -27,6 +28,7 @@ export default function DatasetsAccordion({
   className,
   contentClassName,
   onDatasetsChange,
+  useCloud = false,
 }: DatasetsAccordionProps) {
   const {
     value: isDatasetsPanelOpen,
@@ -41,7 +43,7 @@ export default function DatasetsAccordion({
     removeDataset,
     getDatasetData,
     removeDatasetData,
-  } = useDatasets();
+  } = useDatasets(useCloud);
 
   useEffect(() => {
     if (datasets.length === 0) {
@@ -177,11 +179,11 @@ export default function DatasetsAccordion({
       return;
     }
 
-    return addData(dataset, files)
+    return addData(dataset, files, useCloud)
       .then(async () => {
         await getDatasetData(dataset.id);
 
-        return cognifyDataset(dataset)
+        return cognifyDataset(dataset, useCloud)
           .finally(() => {
             setProcessingDataset(null);
           });
@@ -255,7 +257,6 @@ export default function DatasetsAccordion({
                 closeAccordion={() => toggleDataset(dataset.id)}
                 tools={(
                   <IconButton className="relative">
-                    <input tabIndex={-1} type="file" multiple onChange={handleAddFiles.bind(null, dataset)} className="absolute w-full h-full cursor-pointer opacity-0" />
                     <PopupMenu>
                       <div className="flex flex-col gap-0.5">
                         <div className="hover:bg-gray-100 w-full text-left px-2 cursor-pointer relative">

cognee-frontend/src/app/dashboard/InstanceDatasetsAccordion.tsx

@@ -2,7 +2,7 @@ import { useCallback, useEffect } from "react";
 
 import { fetch, useBoolean } from "@/utils";
 import { checkCloudConnection } from "@/modules/cloud";
-import { CloseIcon, CloudIcon, LocalCogneeIcon } from "@/ui/Icons";
+import { CaretIcon, CloseIcon, CloudIcon, LocalCogneeIcon } from "@/ui/Icons";
 import { CTAButton, GhostButton, IconButton, Input, Modal } from "@/ui/elements";
 
 import DatasetsAccordion, { DatasetsAccordionProps } from "./DatasetsAccordion";
@@ -21,6 +21,7 @@ export default function InstanceDatasetsAccordion({ onDatasetsChange }: Instance
   } = useBoolean(false);
 
   const checkConnectionToCloudCognee = useCallback((apiKey: string) => {
+      fetch.setApiKey(apiKey);
       return checkCloudConnection(apiKey)
         .then(setCloudCogneeConnected)
     }, [setCloudCogneeConnected]);
@@ -71,13 +72,35 @@ export default function InstanceDatasetsAccordion({ onDatasetsChange }: Instance
         onDatasetsChange={onDatasetsChange}
       />
 
-      <button className="w-full flex flex-row items-center justify-between py-1.5 cursor-pointer" onClick={!isCloudCogneeConnected ? openCloudConnectionModal : () => {}}>
+      {isCloudCogneeConnected ? (
-        <div className="flex flex-row items-center gap-2">
+        <DatasetsAccordion
-          <CloudIcon color="#000000" />
+          title={(
-          <span className="text-xs">cloud cognee</span>
+            <div className="flex flex-row items-center justify-between">
-        </div>
+              <div className="flex flex-row items-center gap-2">
-        {isCloudCogneeConnected ? <span className="text-xs text-indigo-600">Connected</span> : <span className="text-xs text-gray-400">Not connected</span>}
+                <LocalCogneeIcon className="text-indigo-700" />
-      </button>
+                <span className="text-xs">cloud cognee</span>
+              </div>
+            </div>
+          )}
+          tools={<span className="text-xs text-indigo-600">Connected</span>}
+          switchCaretPosition={true}
+          className="pt-3 pb-1.5"
+          contentClassName="pl-4"
+          onDatasetsChange={onDatasetsChange}
+          useCloud={true}
+        />
+      ) : (
+        <button className="w-full flex flex-row items-center justify-between py-1.5 cursor-pointer" onClick={!isCloudCogneeConnected ? openCloudConnectionModal : () => {}}>
+          <div className="flex flex-row items-center gap-1.5">
+            <CaretIcon className="rotate-[-90deg]" />
+            <div className="flex flex-row items-center gap-2">
+              <CloudIcon color="#000000" />
+              <span className="text-xs">cloud cognee</span>
+            </div>
+          </div>
+          <span className="text-xs text-gray-400">Not connected</span>
+        </button>
+      )}
 
       <Modal isOpen={isCloudConnectedModalOpen}>
         <div className="w-full max-w-2xl">

cognee-frontend/src/modules/datasets/cognifyDataset.ts

@@ -7,7 +7,7 @@ import { Dataset } from "../ingestion/useDatasets";
 //   edges: { source: string; target: string; label: string }[];
 // }
 
-export default async function cognifyDataset(dataset: Dataset) {
+export default async function cognifyDataset(dataset: Dataset, useCloud: boolean = false) {
   // const data = await (
   return fetch("/v1/cognify", {
     method: "POST",
@@ -18,7 +18,7 @@ export default async function cognifyDataset(dataset: Dataset) {
       datasetIds: [dataset.id],
       runInBackground: false,
     }),
-  })
+  }, useCloud)
   .then((response) => response.json());
   // .then(() => {
   //   return getDatasetGraph(dataset)

cognee-frontend/src/modules/datasets/createDataset.ts

@@ -1,12 +1,12 @@
 import { fetch } from "@/utils";
 
-export default function createDataset(dataset: { name: string }) {
+export default function createDataset(dataset: { name: string }, useCloud = false) {
   return fetch(`/v1/datasets/`, {
     method: "POST",
     body: JSON.stringify(dataset),
     headers: {
       "Content-Type": "application/json",
-    }
+    },
-  })
+  }, useCloud)
     .then((response) => response.json());
 }

cognee-frontend/src/modules/ingestion/addData.ts

@@ -1,19 +1,35 @@
 import { fetch } from "@/utils";
 
-export default function addData(dataset: { id?: string, name?: string }, files: File[]) {
+export default async function addData(dataset: { id?: string, name?: string }, files: File[], useCloud = false) {
-  const formData = new FormData();
+  if (useCloud) {
-  files.forEach((file) => {
+    const data = {
-    formData.append("data", file, file.name);
+      text_data: await Promise.all(files.map(async (file) => file.text())),
-  })
+      datasetId: dataset.id,
-  if (dataset.id) {
+      datasetName: dataset.name,
-    formData.append("datasetId", dataset.id);
+    };
-  }
-  if (dataset.name) {
-    formData.append("datasetName", dataset.name);
-  }
 
-  return fetch("/v1/add", {
+    return fetch("/v1/add", {
-    method: "POST",
+      method: "POST",
-    body: formData,
+      headers: {
-  }).then((response) => response.json());
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(data),
+    }, true).then((response) => response.json());
+  } else {
+    const formData = new FormData();
+    files.forEach((file) => {
+      formData.append("data", file, file.name);
+    })
+    if (dataset.id) {
+      formData.append("datasetId", dataset.id);
+    }
+    if (dataset.name) {
+      formData.append("datasetName", dataset.name);
+    }
+
+    return fetch("/v1/add", {
+      method: "POST",
+      body: formData,
+    }).then((response) => response.json());
+  }
 }

cognee-frontend/src/modules/ingestion/useDatasets.ts

@@ -1,5 +1,4 @@
-import { useCallback, useEffect, useRef, useState } from 'react';
+import { useCallback, useState } from 'react';
-import { v4 } from 'uuid';
 
 import { fetch } from '@/utils';
 import { DataFile } from './useData';
@@ -12,95 +11,96 @@ export interface Dataset {
   status: string;
 }
 
-function useDatasets() {
+function useDatasets(useCloud = false) {
   const [datasets, setDatasets] = useState<Dataset[]>([]);
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  const statusTimeout = useRef<any>(null);
+  // const statusTimeout = useRef<any>(null);
 
-  const fetchDatasetStatuses = useCallback((datasets: Dataset[]) => {
+  // const fetchDatasetStatuses = useCallback((datasets: Dataset[]) => {
-    fetch(
+  //   fetch(
-      `/v1/datasets/status?dataset=${datasets.map(d => d.id).join('&dataset=')}`,
+  //     `/v1/datasets/status?dataset=${datasets.map(d => d.id).join('&dataset=')}`,
-      {
+  //     {
-        headers: {
+  //       headers: {
-          "Content-Type": "application/json",
+  //         "Content-Type": "application/json",
-        },
+  //       },
-      },
+  //     },
-    )
+  //     useCloud,
-      .then((response) => response.json())
+  //   )
-      .then((statuses) => setDatasets(
+  //     .then((response) => response.json())
-        (datasets) => (
+  //     .then((statuses) => setDatasets(
-          datasets.map((dataset) => ({
+  //       (datasets) => (
-            ...dataset,
+  //         datasets.map((dataset) => ({
-            status: statuses[dataset.id]
+  //           ...dataset,
-          }))
+  //           status: statuses[dataset.id]
-      )));
+  //         }))
-  }, []);
+  //     )));
+  // }, [useCloud]);
 
-  const checkDatasetStatuses = useCallback((datasets: Dataset[]) => {
+  // const checkDatasetStatuses = useCallback((datasets: Dataset[]) => {
-    fetchDatasetStatuses(datasets);
+  //   fetchDatasetStatuses(datasets);
 
-    if (statusTimeout.current !== null) {
+  //   if (statusTimeout.current !== null) {
-      clearTimeout(statusTimeout.current);
+  //     clearTimeout(statusTimeout.current);
-    }
+  //   }
 
-    statusTimeout.current = setTimeout(() => {
+  //   statusTimeout.current = setTimeout(() => {
-      checkDatasetStatuses(datasets);
+  //     checkDatasetStatuses(datasets);
-    }, 50000);
+  //   }, 50000);
-  }, [fetchDatasetStatuses]);
+  // }, [fetchDatasetStatuses]);
   
-  useEffect(() => {
+  // useEffect(() => {
-    return () => {
+  //   return () => {
-      if (statusTimeout.current !== null) {
+  //     if (statusTimeout.current !== null) {
-        clearTimeout(statusTimeout.current);
+  //       clearTimeout(statusTimeout.current);
-        statusTimeout.current = null;
+  //       statusTimeout.current = null;
-      }
+  //     }
-    };
+  //   };
-  }, []);
+  // }, []);
 
   const addDataset = useCallback((datasetName: string) => {
-    return createDataset({ name: datasetName  })
+    return createDataset({ name: datasetName  }, useCloud)
       .then((dataset) => {
         setDatasets((datasets) => [
           ...datasets,
           dataset,
         ]);
       });
-  }, []);
+  }, [useCloud]);
 
   const removeDataset = useCallback((datasetId: string) => {
     return fetch(`/v1/datasets/${datasetId}`, {
       method: 'DELETE',
-    })
+    }, useCloud)
       .then(() => {
         setDatasets((datasets) =>
           datasets.filter((dataset) => dataset.id !== datasetId)
         );
       });
-  }, []);
+  }, [useCloud]);
 
   const fetchDatasets = useCallback(() => {
     return fetch('/v1/datasets', {
         headers: {
           "Content-Type": "application/json",
         },
-      })
+      }, useCloud)
       .then((response) => response.json())
       .then((datasets) => {
         setDatasets(datasets);
 
-        if (datasets.length > 0) {
+        // if (datasets.length > 0) {
-          checkDatasetStatuses(datasets);
+        //   checkDatasetStatuses(datasets);
-        }
+        // }
 
         return datasets;
       })
       .catch((error) => {
         console.error('Error fetching datasets:', error);
       });
-  }, [checkDatasetStatuses]);
+  }, [useCloud]);
 
   const getDatasetData = useCallback((datasetId: string) => {
-    return fetch(`/v1/datasets/${datasetId}/data`)
+    return fetch(`/v1/datasets/${datasetId}/data`, {}, useCloud)
       .then((response) => response.json())
       .then((data) => {
         const datasetIndex = datasets.findIndex((dataset) => dataset.id === datasetId);
@@ -118,13 +118,13 @@ function useDatasets() {
 
         return data;
       });
-  }, [datasets]);
+  }, [datasets, useCloud]);
 
   const removeDatasetData = useCallback((datasetId: string, dataId: string) => {
     return fetch(`/v1/datasets/${datasetId}/data/${dataId}`, {
       method: 'DELETE',
-    });
+    }, useCloud);
-  }, []);
+  }, [useCloud]);
 
   return {
     datasets,

cognee-frontend/src/utils/fetch.ts

@@ -6,7 +6,11 @@ const isAuth0Enabled = process.env.USE_AUTH0_AUTHORIZATION?.toLowerCase() === "t
 
 const backendApiUrl = process.env.NEXT_PUBLIC_BACKEND_API_URL || "http://localhost:8000/api";
 
-export default async function fetch(url: string, options: RequestInit = {}): Promise<Response> {
+const cloudApiUrl = process.env.NEXT_PUBLIC_CLOUD_API_URL || "https://api.cognee.ai/api";
+
+let apiKey: string | null = null;
+
+export default async function fetch(url: string, options: RequestInit = {}, useCloud = false): Promise<Response> {
   function retry(lastError: Response) {
     if (!isAuth0Enabled) {
       return Promise.reject(lastError);
@@ -24,10 +28,17 @@ export default async function fetch(url: string, options: RequestInit = {}): Pro
       });
   }
 
-  return global.fetch(backendApiUrl + url, {
+  return global.fetch(
-    ...options,
+    (useCloud ? cloudApiUrl : backendApiUrl) + (useCloud ? url.replace("/v1", "") : url),
-    credentials: "include",
+    {
-  })
+      ...options,
+      headers: {
+        ...options.headers,
+        ...(useCloud ? {"X-Api-Key": apiKey!} : {}),
+      },
+      credentials: "include",
+    },
+  )
     .then((response) => handleServerErrors(response, retry))
     .then((response) => {
       numberOfRetries = 0;
@@ -51,3 +62,7 @@ export default async function fetch(url: string, options: RequestInit = {}): Pro
 fetch.checkHealth = () => {
   return global.fetch(`${backendApiUrl.replace("/api", "")}/health`);
 };
+
+fetch.setApiKey = (newApiKey: string) => {
+  apiKey = newApiKey;
+};