Merge branch 'main' into main

.github/workflows/tests.yml

@@ -193,7 +193,8 @@ jobs:
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
         run: |
-          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} down -v
+          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} down -v || true
+          sudo docker ps -a --filter "label=com.docker.compose.project=${GITHUB_RUN_ID}" -q | xargs -r sudo docker rm -f
 
       - name: Start ragflow:nightly
         run: |
@@ -230,5 +231,9 @@ jobs:
       - name: Stop ragflow:nightly
         if: always()  # always run this step even if previous steps failed
         run: |
-          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} down -v
+          # Sometimes `docker compose down` fail due to hang container, heavy load etc. Need to remove such containers to release resources(for example, listen ports).
-          sudo docker rmi -f ${RAGFLOW_IMAGE:-NO_IMAGE} || true
+          sudo docker compose -f docker/docker-compose.yml -p ${GITHUB_RUN_ID} down -v || true
+          sudo docker ps -a --filter "label=com.docker.compose.project=${GITHUB_RUN_ID}" -q | xargs -r sudo docker rm -f
+          if [[ -n ${RAGFLOW_IMAGE} ]]; then
+            sudo docker rmi -f ${RAGFLOW_IMAGE}
+          fi

api/apps/kb_app.py

@@ -122,7 +122,7 @@ def update():
         if not e:
             return get_data_error_result(
                 message="Database error (Knowledgebase rename)!")
-        errors = Connector2KbService.link_connectors(kb.id, [conn["id"] for conn in connectors], current_user.id)
+        errors = Connector2KbService.link_connectors(kb.id, [conn for conn in connectors], current_user.id)
         if errors:
             logging.error("Link KB errors: ", errors)
         kb = kb.to_dict()

api/db/services/connector_service.py

@@ -73,7 +73,9 @@ class ConnectorService(CommonService):
             return
         SyncLogsService.filter_delete([SyncLogs.connector_id==connector_id, SyncLogs.kb_id==kb_id])
         docs = DocumentService.query(source_type=f"{conn.source}/{conn.id}")
-        return FileService.delete_docs([d.id for d in docs], tenant_id)
+        err = FileService.delete_docs([d.id for d in docs], tenant_id)
+        SyncLogsService.schedule(connector_id, kb_id, reindex=True)
+        return err
 
 
 class SyncLogsService(CommonService):
@@ -226,16 +228,20 @@ class Connector2KbService(CommonService):
     model = Connector2Kb
 
     @classmethod
-    def link_connectors(cls, kb_id:str, connector_ids: list[str], tenant_id:str):
+    def link_connectors(cls, kb_id:str, connectors: list[dict], tenant_id:str):
         arr = cls.query(kb_id=kb_id)
         old_conn_ids = [a.connector_id for a in arr]
-        for conn_id in connector_ids:
+        connector_ids = []
+        for conn in connectors:
+            conn_id = conn["id"]
+            connector_ids.append(conn_id)
             if conn_id in old_conn_ids:
                 continue
             cls.save(**{
                 "id": get_uuid(),
                 "connector_id": conn_id,
-                "kb_id": kb_id
+                "kb_id": kb_id,
+                "auto_parse": conn.get("auto_parse", "1")
             })
             SyncLogsService.schedule(conn_id, kb_id, reindex=True)
 

common/data_source/discord_connector.py

@@ -63,7 +63,7 @@ def _convert_message_to_document(
         semantic_identifier=semantic_identifier,
         doc_updated_at=doc_updated_at,
         blob=message.content.encode("utf-8"),
-        extension="txt",
+        extension=".txt",
         size_bytes=len(message.content.encode("utf-8")),
     )
 
@@ -275,7 +275,7 @@ class DiscordConnector(LoadConnector, PollConnector):
                 semantic_identifier=f"{min_updated_at} -> {max_updated_at}",
                 doc_updated_at=max_updated_at,
                 blob=blob,
-                extension="txt",
+                extension=".txt",
                 size_bytes=size_bytes,
             )
 

common/data_source/notion_connector.py

@@ -1,6 +1,5 @@
 import logging
 from collections.abc import Generator
-from datetime import datetime, timezone
 from typing import Any, Optional
 from retry import retry
 
@@ -33,7 +32,7 @@ from common.data_source.utils import (
     batch_generator,
     fetch_notion_data,
     properties_to_str,
-    filter_pages_by_time
+    filter_pages_by_time, datetime_from_string
 )
 
 
@@ -293,9 +292,9 @@ class NotionConnector(LoadConnector, PollConnector):
                 blob=blob,
                 source=DocumentSource.NOTION,
                 semantic_identifier=page_title,
-                extension="txt",
+                extension=".txt",
                 size_bytes=len(blob),
-                doc_updated_at=datetime.fromisoformat(page.last_edited_time).astimezone(timezone.utc)
+                doc_updated_at=datetime_from_string(page.last_edited_time)
             )
 
         if self.recursive_index_enabled and all_child_page_ids:

rag/svr/sync_data_source.py

@@ -63,6 +63,8 @@ class SyncBase:
                     if task["poll_range_start"]:
                         next_update = task["poll_range_start"]
                     for document_batch in document_batch_generator:
+                        if not document_batch:
+                            continue
                         min_update = min([doc.doc_updated_at for doc in document_batch])
                         max_update = max([doc.doc_updated_at for doc in document_batch])
                         next_update = max([next_update, max_update])

sdk/python/ragflow_sdk/modules/session.py

@@ -74,15 +74,15 @@ class Session(Base):
                 json_data = res.json()
             except ValueError:
                 raise Exception(f"Invalid response {res}")
-            yield self._structure_answer(json_data)
+            yield self._structure_answer(json_data["data"])
         
 
     def _structure_answer(self, json_data):
         if self.__session_type == "agent":
-           answer = json_data["data"]["data"]["content"]
+           answer = json_data["data"]["content"]
         elif self.__session_type == "chat":
-            answer =json_data["data"]["answer"]
+            answer = json_data["answer"]
-        reference = json_data["data"].get("reference", {})
+        reference = json_data.get("reference", {})
         temp_dict = {
             "content": answer,
             "role": "assistant"

web/src/pages/agent/constant/index.tsx

@@ -598,7 +598,7 @@ export const initialDataOperationsValues = {
 
 export const initialVariableAssignerValues = {};
 
-export const initialVariableAggregatorValues = {};
+export const initialVariableAggregatorValues = { outputs: {}, groups: [] };
 
 export const CategorizeAnchorPointPositions = [
   { top: 1, right: 34 },

web/src/pages/agent/form/components/prompt-editor/variable-picker-plugin.tsx

@@ -36,6 +36,7 @@ import {
   useShowSecondaryMenu,
 } from '@/pages/agent/hooks/use-build-structured-output';
 import { useFilterQueryVariableOptionsByTypes } from '@/pages/agent/hooks/use-get-begin-query';
+import { get } from 'lodash';
 import { PromptIdentity } from '../../agent-form/use-build-prompt-options';
 import { StructuredOutputSecondaryMenu } from '../structured-output-secondary-menu';
 import { ProgrammaticTag } from './constant';
@@ -45,18 +46,21 @@ class VariableInnerOption extends MenuOption {
   value: string;
   parentLabel: string | JSX.Element;
   icon?: ReactNode;
+  type?: string;
 
   constructor(
     label: string,
     value: string,
     parentLabel: string | JSX.Element,
     icon?: ReactNode,
+    type?: string,
   ) {
     super(value);
     this.label = label;
     this.value = value;
     this.parentLabel = parentLabel;
     this.icon = icon;
+    this.type = type;
   }
 }
 
@@ -126,9 +130,10 @@ function VariablePickerMenuItem({
               <li
                 key={x.value}
                 onClick={() => selectOptionAndCleanUp(x)}
-                className="hover:bg-bg-card p-1 text-text-primary rounded-sm"
+                className="hover:bg-bg-card p-1 text-text-primary rounded-sm flex justify-between items-center"
               >
-                {x.label}
+                <span className="truncate flex-1 min-w-0">{x.label}</span>
+                <span className="text-text-secondary">{get(x, 'type')}</span>
               </li>
             );
           })}
@@ -146,6 +151,7 @@ export type VariablePickerMenuOptionType = {
     label: string;
     value: string;
     icon: ReactNode;
+    type?: string;
   }>;
 };
 
@@ -214,7 +220,13 @@ export default function VariablePickerMenuPlugin({
           x.label,
           x.title,
           x.options.map((y) => {
-            return new VariableInnerOption(y.label, y.value, x.label, y.icon);
+            return new VariableInnerOption(
+              y.label,
+              y.value,
+              x.label,
+              y.icon,
+              y.type,
+            );
           }),
         ),
     );
@@ -378,7 +390,7 @@ export default function VariablePickerMenuPlugin({
         const nextOptions = buildNextOptions();
         return anchorElementRef.current && nextOptions.length
           ? ReactDOM.createPortal(
-              <div className="typeahead-popover w-[200px] p-2 bg-bg-base">
+              <div className="typeahead-popover w-80 p-2 bg-bg-base">
                 <ul className="scroll-auto overflow-x-hidden">
                   {nextOptions.map((option, i: number) => (
                     <VariablePickerMenuItem

web/src/pages/agent/form/components/query-variable.tsx

@@ -18,6 +18,7 @@ type QueryVariableProps = {
   label?: ReactNode;
   hideLabel?: boolean;
   className?: string;
+  onChange?: (value: string) => void;
 };
 
 export function QueryVariable({
@@ -26,6 +27,7 @@ export function QueryVariable({
   label,
   hideLabel = false,
   className,
+  onChange,
 }: QueryVariableProps) {
   const { t } = useTranslation();
   const form = useFormContext();
@@ -46,7 +48,11 @@ export function QueryVariable({
           <FormControl>
             <GroupedSelectWithSecondaryMenu
               options={finalOptions}
-              {...field}
+              value={field.value}
+              onChange={(val) => {
+                field.onChange(val);
+                onChange?.(val);
+              }}
               // allowClear
               types={types}
             ></GroupedSelectWithSecondaryMenu>

web/src/pages/agent/form/components/select-with-secondary-menu.tsx

@@ -209,8 +209,12 @@ export function GroupedSelectWithSecondaryMenu({
                         onChange?.(option.value);
                         setOpen(false);
                       }}
+                      className="flex items-center justify-between"
                     >
-                      {option.label}
+                      <span> {option.label}</span>
+                      <span className="text-text-secondary">
+                        {get(option, 'type')}
+                      </span>
                     </CommandItem>
                   );
                 })}

web/src/pages/agent/form/components/structured-output-secondary-menu.tsx

@@ -112,9 +112,12 @@ export function StructuredOutputSecondaryMenu({
       <HoverCardTrigger asChild>
         <li
           onClick={handleMenuClick}
-          className="hover:bg-bg-card py-1 px-2 text-text-primary rounded-sm text-sm flex justify-between items-center"
+          className="hover:bg-bg-card py-1 px-2 text-text-primary rounded-sm text-sm flex justify-between items-center gap-2"
         >
-          {data.label} <ChevronRight className="size-3.5 text-text-secondary" />
+          <div className="flex justify-between flex-1">
+            {data.label} <span className="text-text-secondary">object</span>
+          </div>
+          <ChevronRight className="size-3.5 text-text-secondary" />
         </li>
       </HoverCardTrigger>
       <HoverCardContent

web/src/pages/agent/form/variable-aggregator-form/dynamic-group-variable.tsx

@@ -5,6 +5,7 @@ import { Plus, Trash2 } from 'lucide-react';
 import { useFieldArray, useFormContext } from 'react-hook-form';
 import { useGetVariableLabelOrTypeByValue } from '../../hooks/use-get-begin-query';
 import { QueryVariable } from '../components/query-variable';
+import { NameInput } from './name-input';
 
 type DynamicGroupVariableProps = {
   name: string;
@@ -36,9 +37,17 @@ export function DynamicGroupVariable({
       <div className="flex items-center justify-between">
         <div className="flex items-center gap-3">
           <RAGFlowFormItem name={`${name}.group_name`} className="w-32">
+            {(field) => (
+              <NameInput
+                value={field.value}
+                onChange={field.onChange}
+              ></NameInput>
+            )}
+          </RAGFlowFormItem>
+          {/* Use a hidden form to store data types; otherwise, data loss may occur. */}
+          <RAGFlowFormItem name={`${name}.type`} className="hidden">
             <Input></Input>
           </RAGFlowFormItem>
-
           <Button
             variant={'ghost'}
             type="button"
@@ -72,6 +81,14 @@ export function DynamicGroupVariable({
               className="flex-1 min-w-0"
               hideLabel
               types={firstType && fields.length > 1 ? [firstType] : []}
+              onChange={(val) => {
+                const type = getType(val);
+                if (type && index === 0) {
+                  form.setValue(`${name}.type`, type, {
+                    shouldDirty: true,
+                  });
+                }
+              }}
             ></QueryVariable>
             <Button
               variant={'ghost'}

web/src/pages/agent/form/variable-aggregator-form/index.tsx

@@ -2,41 +2,27 @@ import { BlockButton } from '@/components/ui/button';
 import { Form } from '@/components/ui/form';
 import { Separator } from '@/components/ui/separator';
 import { zodResolver } from '@hookform/resolvers/zod';
-import { memo } from 'react';
+import { memo, useCallback } from 'react';
 import { useFieldArray, useForm } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
-import { z } from 'zod';
 import { initialDataOperationsValues } from '../../constant';
 import { useFormValues } from '../../hooks/use-form-values';
-import { useWatchFormChange } from '../../hooks/use-watch-form-change';
 import { INextOperatorForm } from '../../interface';
+import useGraphStore from '../../store';
 import { buildOutputList } from '../../utils/build-output-list';
 import { FormWrapper } from '../components/form-wrapper';
 import { Output } from '../components/output';
 import { DynamicGroupVariable } from './dynamic-group-variable';
-
+import { FormSchema, VariableAggregatorFormSchemaType } from './schema';
-export const RetrievalPartialSchema = {
+import { useWatchFormChange } from './use-watch-change';
-  groups: z.array(
-    z.object({
-      group_name: z.string(),
-      variables: z.array(z.object({ value: z.string().optional() })),
-    }),
-  ),
-  operations: z.string(),
-};
-
-export const FormSchema = z.object(RetrievalPartialSchema);
-
-export type DataOperationsFormSchemaType = z.infer<typeof FormSchema>;
-
-const outputList = buildOutputList(initialDataOperationsValues.outputs);
 
 function VariableAggregatorForm({ node }: INextOperatorForm) {
   const { t } = useTranslation();
+  const getNode = useGraphStore((state) => state.getNode);
 
   const defaultValues = useFormValues(initialDataOperationsValues, node);
 
-  const form = useForm<DataOperationsFormSchemaType>({
+  const form = useForm<VariableAggregatorFormSchemaType>({
     defaultValues: defaultValues,
     mode: 'onChange',
     resolver: zodResolver(FormSchema),
@@ -48,7 +34,15 @@ function VariableAggregatorForm({ node }: INextOperatorForm) {
     control: form.control,
   });
 
-  useWatchFormChange(node?.id, form, true);
+  const appendItem = useCallback(() => {
+    append({ group_name: `Group ${fields.length}`, variables: [] });
+  }, [append, fields.length]);
+
+  const outputList = buildOutputList(
+    getNode(node?.id)?.data.form.outputs ?? {},
+  );
+
+  useWatchFormChange(node?.id, form);
 
   return (
     <Form {...form}>
@@ -63,16 +57,10 @@ function VariableAggregatorForm({ node }: INextOperatorForm) {
             ></DynamicGroupVariable>
           ))}
         </section>
-        <BlockButton
+        <BlockButton onClick={appendItem}>{t('common.add')}</BlockButton>
-          onClick={() =>
-            append({ group_name: `Group ${fields.length}`, variables: [] })
-          }
-        >
-          {t('common.add')}
-        </BlockButton>
         <Separator />
 
-        <Output list={outputList} isFormRequired></Output>
+        <Output list={outputList}></Output>
       </FormWrapper>
     </Form>
   );

web/src/pages/agent/form/variable-aggregator-form/name-input.tsx

@@ -0,0 +1,55 @@
+import { Input } from '@/components/ui/input';
+import { PenLine } from 'lucide-react';
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { useHandleNameChange } from './use-handle-name-change';
+
+type NameInputProps = {
+  value: string;
+  onChange: (value: string) => void;
+};
+
+export function NameInput({ value, onChange }: NameInputProps) {
+  const { name, handleNameBlur, handleNameChange } = useHandleNameChange(value);
+  const inputRef = useRef<HTMLInputElement>(null);
+
+  const [isEditingMode, setIsEditingMode] = useState(false);
+
+  const switchIsEditingMode = useCallback(() => {
+    setIsEditingMode((prev) => !prev);
+  }, []);
+
+  const handleBlur = useCallback(() => {
+    const nextName = handleNameBlur();
+    setIsEditingMode(false);
+    onChange(nextName);
+  }, [handleNameBlur, onChange]);
+
+  useEffect(() => {
+    if (isEditingMode && inputRef.current) {
+      requestAnimationFrame(() => {
+        inputRef.current?.focus();
+      });
+    }
+  }, [isEditingMode]);
+
+  return (
+    <div className="flex items-center gap-1 flex-1">
+      {isEditingMode ? (
+        <Input
+          ref={inputRef}
+          value={name}
+          onBlur={handleBlur}
+          onChange={handleNameChange}
+        ></Input>
+      ) : (
+        <div className="flex items-center justify-between gap-2 text-base w-full">
+          <span className="truncate flex-1">{name}</span>
+          <PenLine
+            onClick={switchIsEditingMode}
+            className="size-3.5 text-text-secondary cursor-pointer hidden group-hover:block"
+          />
+        </div>
+      )}
+    </div>
+  );
+}

web/src/pages/agent/form/variable-aggregator-form/schema.ts

@@ -0,0 +1,15 @@
+import { z } from 'zod';
+
+export const VariableAggregatorSchema = {
+  groups: z.array(
+    z.object({
+      group_name: z.string(),
+      variables: z.array(z.object({ value: z.string().optional() })),
+      type: z.string().optional(),
+    }),
+  ),
+};
+
+export const FormSchema = z.object(VariableAggregatorSchema);
+
+export type VariableAggregatorFormSchemaType = z.infer<typeof FormSchema>;

web/src/pages/agent/form/variable-aggregator-form/use-handle-name-change.ts

@@ -0,0 +1,37 @@
+import message from '@/components/ui/message';
+import { trim } from 'lodash';
+import { ChangeEvent, useCallback, useEffect, useState } from 'react';
+import { useFormContext } from 'react-hook-form';
+import { VariableAggregatorFormSchemaType } from './schema';
+
+export const useHandleNameChange = (previousName: string) => {
+  const [name, setName] = useState<string>('');
+  const form = useFormContext<VariableAggregatorFormSchemaType>();
+
+  const handleNameBlur = useCallback(() => {
+    const names = form.getValues('groups');
+    const existsSameName = names.some((x) => x.group_name === name);
+    if (trim(name) === '' || existsSameName) {
+      if (existsSameName && previousName !== name) {
+        message.error('The name cannot be repeated');
+      }
+      setName(previousName);
+      return previousName;
+    }
+    return name;
+  }, [form, name, previousName]);
+
+  const handleNameChange = useCallback((e: ChangeEvent<any>) => {
+    setName(e.target.value);
+  }, []);
+
+  useEffect(() => {
+    setName(previousName);
+  }, [previousName]);
+
+  return {
+    name,
+    handleNameBlur,
+    handleNameChange,
+  };
+};

web/src/pages/agent/form/variable-aggregator-form/use-watch-change.ts

@@ -0,0 +1,31 @@
+import { useEffect } from 'react';
+import { UseFormReturn, useWatch } from 'react-hook-form';
+import useGraphStore from '../../store';
+import { VariableAggregatorFormSchemaType } from './schema';
+
+export function useWatchFormChange(
+  id?: string,
+  form?: UseFormReturn<VariableAggregatorFormSchemaType>,
+) {
+  let values = useWatch({ control: form?.control });
+  const { replaceNodeForm } = useGraphStore((state) => state);
+
+  useEffect(() => {
+    if (id && form?.formState.isDirty) {
+      const outputs = values.groups?.reduce(
+        (pre, cur) => {
+          if (cur.group_name) {
+            pre[cur.group_name] = {
+              type: cur.type,
+            };
+          }
+
+          return pre;
+        },
+        {} as Record<string, Record<string, any>>,
+      );
+
+      replaceNodeForm(id, { ...values, outputs: outputs ?? {} });
+    }
+  }, [form?.formState.isDirty, id, replaceNodeForm, values]);
+}