diff --git a/.env.example b/.env.example index b4b1b88b..081c9026 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,14 @@ # Set to true to disable Langflow ingestion and use traditional OpenRAG processor # If unset or false, Langflow pipeline will be used (default: upload -> ingest -> delete) DISABLE_INGEST_WITH_LANGFLOW=false + +# Langflow HTTP timeout configuration (in seconds) +# For large documents (300+ pages), ingestion can take 30+ minutes +# Increase these values if you experience timeouts with very large PDFs +# Default: 2400 seconds (40 minutes) total timeout, 30 seconds connection timeout +# LANGFLOW_TIMEOUT=2400 +# LANGFLOW_CONNECT_TIMEOUT=30 + # make one like so https://docs.langflow.org/api-keys-and-authentication#langflow-secret-key LANGFLOW_SECRET_KEY= diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..3b871ae9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 + +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + commit-message: + prefix: "build(deps):" + include: scope + diff --git a/.github/workflows/update-uv-lock.yml b/.github/workflows/update-uv-lock.yml new file mode 100644 index 00000000..2ebc8ea5 --- /dev/null +++ b/.github/workflows/update-uv-lock.yml @@ -0,0 +1,52 @@ +name: Update uv.lock on version bump + +on: + push: + branches: + - main + paths: + - 'pyproject.toml' + workflow_dispatch: + +jobs: + update-lock: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Update uv.lock + run: uv sync + + - name: Check for changes + id: changes + run: | + if git diff --quiet uv.lock; then + echo "changed=false" >> $GITHUB_OUTPUT + echo "No changes to uv.lock" + else + echo "changed=true" >> $GITHUB_OUTPUT + echo "uv.lock has been updated" + fi + + - name: Commit and push uv.lock + if: steps.changes.outputs.changed == 'true' + run: | + git config --local user.email "github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git add uv.lock + git commit -m "chore: update uv.lock after version bump [skip ci]" + git push + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..b6c7a6fc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,7 @@ +repos: + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: ["--baseline", ".secrets.baseline", "--exclude-lines", "code_hash"] + diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 00000000..28837d45 --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,180 @@ +{ + "version": "1.5.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "OpenAIDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TelegramBotTokenDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_baseline_file", + "filename": ".secrets.baseline" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + }, + { + "path": "detect_secrets.filters.regex.should_exclude_file", + "pattern": [ + "flows/.*\\.json$" + ] + }, + { + "path": "detect_secrets.filters.regex.should_exclude_line", + "pattern": [ + "code_hash" + ] + } + ], + "results": { + "docs/docs/_partial-integrate-chat.mdx": [ + { + "type": "Secret Keyword", + "filename": "docs/docs/_partial-integrate-chat.mdx", + "hashed_secret": "e42fd8b9ad15d8fa5f4718cad7cf19b522807996", + "is_verified": false, + "line_number": 30 + } + ], + "src/main.py": [ + { + "type": "Base64 High Entropy String", + "filename": "src/main.py", + "hashed_secret": "131a83e9ef8660d7dd0771da7ce5954d9ea801ee", + "is_verified": false, + "line_number": 404 + } + ], + "src/models/processors.py": [ + { + "type": "Base64 High Entropy String", + "filename": "src/models/processors.py", + "hashed_secret": "131a83e9ef8660d7dd0771da7ce5954d9ea801ee", + "is_verified": false, + "line_number": 763 + } + ], + "src/services/langflow_file_service.py": [ + { + "type": "Base64 High Entropy String", + "filename": "src/services/langflow_file_service.py", + "hashed_secret": "131a83e9ef8660d7dd0771da7ce5954d9ea801ee", + "is_verified": false, + "line_number": 97 + } + ] + }, + "generated_at": "2025-12-09T20:33:13Z" +} diff --git a/flows/ingestion_flow.json b/flows/ingestion_flow.json index 25a5cefd..5d512b57 100644 --- a/flows/ingestion_flow.json +++ b/flows/ingestion_flow.json @@ -667,7 +667,7 @@ ], "frozen": false, "icon": "braces", - "last_updated": "2025-12-03T21:41:00.148Z", + "last_updated": "2025-12-12T20:12:18.129Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": {}, @@ -717,7 +717,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "79455c62-cdb1-4f14-bf44-8e76acc020a6" + "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" }, "_type": "Component", "code": { @@ -1399,7 +1399,7 @@ "description": "Uses Docling to process input documents connecting to your instance of Docling Serve.", "display_name": "Docling Serve", "documentation": "https://docling-project.github.io/docling/", - "edited": false, + "edited": true, "field_order": [ "path", "file_path", @@ -1417,9 +1417,8 @@ "frozen": false, "icon": "Docling", "legacy": false, - "lf_version": "1.7.0.dev21", "metadata": { - "code_hash": "26eeb513dded", + "code_hash": "5723576d00e5", "dependencies": { "dependencies": [ { @@ -1428,20 +1427,20 @@ }, { "name": "docling_core", - "version": "2.48.4" + "version": "2.49.0" }, { "name": "pydantic", - "version": "2.10.6" + "version": "2.11.10" }, { "name": "lfx", - "version": "0.1.12.dev31" + "version": "0.2.0.dev21" } ], "total_dependencies": 4 }, - "module": "lfx.components.docling.docling_remote.DoclingRemoteComponent" + "module": "custom_components.docling_serve" }, "minimized": false, "output_types": [], @@ -1451,8 +1450,12 @@ "cache": true, "display_name": "Files", "group_outputs": false, + "hidden": null, + "loop_types": null, "method": "load_files", "name": "dataframe", + "options": null, + "required_inputs": null, "selected": "DataFrame", "tool_mode": true, "types": [ @@ -1473,6 +1476,7 @@ "list": false, "list_add_label": "Add More", "name": "api_headers", + "override_skip": false, "placeholder": "", "required": false, "show": true, @@ -1480,6 +1484,7 @@ "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, + "track_in_telemetry": false, "type": "NestedDict", "value": {} }, @@ -1493,12 +1498,14 @@ "list_add_label": "Add More", "load_from_db": false, "name": "api_url", + "override_skip": false, "placeholder": "", "required": true, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, + "track_in_telemetry": false, "type": "str", "value": "http://localhost:5001" }, @@ -1518,7 +1525,7 @@ "show": true, "title_case": false, "type": "code", - "value": "import base64\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom pydantic import ValidationError\n\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data\nfrom lfx.utils.util import transform_localhost_url\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n ),\n NestedDictInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional dictionary of additional headers required for connecting to Docling Serve.\"),\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:\n # Transform localhost URLs to container-accessible hosts when running in a container\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(client: httpx.Client, file_path: Path, options: dict[str, Any]) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [{\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n # Check if processing exceeds the maximum poll timeout\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n # Call for a new status update\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task['task_id']}\")\n\n # Check if the status call gets into 5xx errors and retry\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(f\"The status requests got a http response {response.status_code} too many times.\")\n return None\n continue\n\n # Update task status\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task['task_id']}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n return Data(data={\"doc\": doc, \"file_path\": str(file_path)})\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = []\n with (\n httpx.Client(headers=self.api_headers) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is None:\n processed_data.append(None)\n continue\n\n futures.append((i, executor.submit(_convert_document, client, file.path, docling_options)))\n\n for _index, future in futures:\n try:\n result_data = future.result()\n processed_data.append(result_data)\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n" + "value": "import base64\nimport time\nfrom concurrent.futures import Future, ThreadPoolExecutor\nfrom pathlib import Path\nfrom typing import Any\n\nimport httpx\nfrom docling_core.types.doc import DoclingDocument\nfrom pydantic import ValidationError\n\nfrom lfx.base.data import BaseFileComponent\nfrom lfx.inputs import IntInput, NestedDictInput, StrInput\nfrom lfx.inputs.inputs import FloatInput\nfrom lfx.schema import Data\nfrom lfx.utils.util import transform_localhost_url\n\n\nclass DoclingRemoteComponent(BaseFileComponent):\n display_name = \"Docling Serve\"\n description = \"Uses Docling to process input documents connecting to your instance of Docling Serve.\"\n documentation = \"https://docling-project.github.io/docling/\"\n trace_type = \"tool\"\n icon = \"Docling\"\n name = \"DoclingRemote\"\n\n MAX_500_RETRIES = 5\n\n # https://docling-project.github.io/docling/usage/supported_formats/\n VALID_EXTENSIONS = [\n \"adoc\",\n \"asciidoc\",\n \"asc\",\n \"bmp\",\n \"csv\",\n \"dotx\",\n \"dotm\",\n \"docm\",\n \"docx\",\n \"htm\",\n \"html\",\n \"jpeg\",\n \"jpg\",\n \"json\",\n \"md\",\n \"pdf\",\n \"png\",\n \"potx\",\n \"ppsx\",\n \"pptm\",\n \"potm\",\n \"ppsm\",\n \"pptx\",\n \"tiff\",\n \"txt\",\n \"xls\",\n \"xlsx\",\n \"xhtml\",\n \"xml\",\n \"webp\",\n ]\n\n inputs = [\n *BaseFileComponent.get_base_inputs(),\n StrInput(\n name=\"api_url\",\n display_name=\"Server address\",\n info=\"URL of the Docling Serve instance.\",\n required=True,\n ),\n IntInput(\n name=\"max_concurrency\",\n display_name=\"Concurrency\",\n info=\"Maximum number of concurrent requests for the server.\",\n advanced=True,\n value=2,\n ),\n FloatInput(\n name=\"max_poll_timeout\",\n display_name=\"Maximum poll time\",\n info=\"Maximum waiting time for the document conversion to complete.\",\n advanced=True,\n value=3600,\n ),\n NestedDictInput(\n name=\"api_headers\",\n display_name=\"HTTP headers\",\n advanced=True,\n required=False,\n info=(\"Optional dictionary of additional headers required for connecting to Docling Serve.\"),\n ),\n NestedDictInput(\n name=\"docling_serve_opts\",\n display_name=\"Docling options\",\n advanced=True,\n required=False,\n info=(\n \"Optional dictionary of additional options. \"\n \"See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.\"\n ),\n ),\n ]\n\n outputs = [\n *BaseFileComponent.get_base_outputs(),\n ]\n\n def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:\n # Transform localhost URLs to container-accessible hosts when running in a container\n transformed_url = transform_localhost_url(self.api_url)\n base_url = f\"{transformed_url}/v1\"\n\n def _convert_document(client: httpx.Client, file_path: Path, options: dict[str, Any]) -> Data | None:\n encoded_doc = base64.b64encode(file_path.read_bytes()).decode()\n payload = {\n \"options\": options,\n \"sources\": [{\"kind\": \"file\", \"base64_string\": encoded_doc, \"filename\": file_path.name}],\n }\n\n response = client.post(f\"{base_url}/convert/source/async\", json=payload)\n response.raise_for_status()\n task = response.json()\n\n http_failures = 0\n retry_status_start = 500\n retry_status_end = 600\n start_wait_time = time.monotonic()\n while task[\"task_status\"] not in (\"success\", \"failure\"):\n # Check if processing exceeds the maximum poll timeout\n processing_time = time.monotonic() - start_wait_time\n if processing_time >= self.max_poll_timeout:\n msg = (\n f\"Processing time {processing_time=} exceeds the maximum poll timeout {self.max_poll_timeout=}.\"\n \"Please increase the max_poll_timeout parameter or review why the processing \"\n \"takes long on the server.\"\n )\n self.log(msg)\n raise RuntimeError(msg)\n\n # Call for a new status update\n time.sleep(2)\n response = client.get(f\"{base_url}/status/poll/{task['task_id']}\")\n\n # Check if the status call gets into 5xx errors and retry\n if retry_status_start <= response.status_code < retry_status_end:\n http_failures += 1\n if http_failures > self.MAX_500_RETRIES:\n self.log(f\"The status requests got a http response {response.status_code} too many times.\")\n return None\n continue\n\n # Update task status\n task = response.json()\n\n result_resp = client.get(f\"{base_url}/result/{task['task_id']}\")\n result_resp.raise_for_status()\n result = result_resp.json()\n\n if \"json_content\" not in result[\"document\"] or result[\"document\"][\"json_content\"] is None:\n self.log(\"No JSON DoclingDocument found in the result.\")\n return None\n\n try:\n doc = DoclingDocument.model_validate(result[\"document\"][\"json_content\"])\n return Data(data={\"doc\": doc, \"file_path\": str(file_path)})\n except ValidationError as e:\n self.log(f\"Error validating the document. {e}\")\n return None\n\n docling_options = {\n \"to_formats\": [\"json\"],\n \"image_export_mode\": \"placeholder\",\n **(self.docling_serve_opts or {}),\n }\n\n processed_data: list[Data | None] = []\n with (\n httpx.Client(headers=self.api_headers) as client,\n ThreadPoolExecutor(max_workers=self.max_concurrency) as executor,\n ):\n futures: list[tuple[int, Future]] = []\n for i, file in enumerate(file_list):\n if file.path is None:\n processed_data.append(None)\n continue\n\n futures.append((i, executor.submit(_convert_document, client, file.path, docling_options)))\n\n for _index, future in futures:\n try:\n result_data = future.result()\n processed_data.append(result_data)\n except (httpx.HTTPStatusError, httpx.RequestError, KeyError, ValueError) as exc:\n self.log(f\"Docling remote processing failed: {exc}\")\n raise\n\n return self.rollup_data(file_list, processed_data)\n" }, "delete_server_file_after_processing": { "_input_type": "BoolInput", @@ -1529,24 +1536,28 @@ "list": false, "list_add_label": "Add More", "name": "delete_server_file_after_processing", + "override_skip": false, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, + "track_in_telemetry": true, "type": "bool", "value": true }, "docling_serve_opts": { "_input_type": "NestedDictInput", - "advanced": false, + "advanced": true, "display_name": "Docling options", "dynamic": false, "info": "Optional dictionary of additional options. See https://github.com/docling-project/docling-serve/blob/main/docs/usage.md for more information.", "list": false, "list_add_label": "Add More", + "load_from_db": false, "name": "docling_serve_opts", + "override_skip": false, "placeholder": "", "required": false, "show": true, @@ -1554,6 +1565,7 @@ "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, + "track_in_telemetry": false, "type": "NestedDict", "value": { "do_ocr": false, @@ -1580,11 +1592,13 @@ "list": true, "list_add_label": "Add More", "name": "file_path", + "override_skip": false, "placeholder": "", "required": false, "show": true, "title_case": false, "trace_as_metadata": true, + "track_in_telemetry": false, "type": "other", "value": "" }, @@ -1597,12 +1611,14 @@ "list": false, "list_add_label": "Add More", "name": "ignore_unspecified_files", + "override_skip": false, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, + "track_in_telemetry": true, "type": "bool", "value": false }, @@ -1615,30 +1631,34 @@ "list": false, "list_add_label": "Add More", "name": "ignore_unsupported_extensions", + "override_skip": false, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, + "track_in_telemetry": true, "type": "bool", "value": true }, "max_concurrency": { "_input_type": "IntInput", - "advanced": false, + "advanced": true, "display_name": "Concurrency", "dynamic": false, "info": "Maximum number of concurrent requests for the server.", "list": false, "list_add_label": "Add More", "name": "max_concurrency", + "override_skip": false, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, + "track_in_telemetry": true, "type": "int", "value": 2 }, @@ -1651,12 +1671,14 @@ "list": false, "list_add_label": "Add More", "name": "max_poll_timeout", + "override_skip": false, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, + "track_in_telemetry": true, "type": "float", "value": 3600 }, @@ -1678,6 +1700,7 @@ "htm", "html", "jpeg", + "jpg", "json", "md", "pdf", @@ -1702,16 +1725,19 @@ "gz" ], "file_path": [], - "info": "Supported file extensions: adoc, asciidoc, asc, bmp, csv, dotx, dotm, docm, docx, htm, html, jpeg, json, md, pdf, png, potx, ppsx, pptm, potm, ppsm, pptx, tiff, txt, xls, xlsx, xhtml, xml, webp; optionally bundled in file extensions: zip, tar, tgz, bz2, gz", + "info": "Supported file extensions: adoc, asciidoc, asc, bmp, csv, dotx, dotm, docm, docx, htm, html, jpeg, jpg, json, md, pdf, png, potx, ppsx, pptm, potm, ppsm, pptx, tiff, txt, xls, xlsx, xhtml, xml, webp; optionally bundled in file extensions: zip, tar, tgz, bz2, gz", "list": true, "list_add_label": "Add More", "name": "path", + "override_skip": false, "placeholder": "", "required": false, "show": true, "temp_file": false, "title_case": false, + "tool_mode": true, "trace_as_metadata": true, + "track_in_telemetry": false, "type": "file", "value": "" }, @@ -1725,12 +1751,14 @@ "list_add_label": "Add More", "load_from_db": false, "name": "separator", + "override_skip": false, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, + "track_in_telemetry": false, "type": "str", "value": "\n\n" }, @@ -1743,12 +1771,14 @@ "list": false, "list_add_label": "Add More", "name": "silent_errors", + "override_skip": false, "placeholder": "", "required": false, "show": true, "title_case": false, "tool_mode": false, "trace_as_metadata": true, + "track_in_telemetry": true, "type": "bool", "value": false } @@ -1761,7 +1791,7 @@ "dragging": false, "id": "DoclingRemote-Dp3PX", "measured": { - "height": 475, + "height": 312, "width": 320 }, "position": { @@ -2060,7 +2090,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-12-03T21:41:00.319Z", + "last_updated": "2025-12-12T20:12:18.208Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -2107,7 +2137,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "79455c62-cdb1-4f14-bf44-8e76acc020a6" + "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" }, "_type": "Component", "ascending": { @@ -2511,7 +2541,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-12-03T21:41:00.320Z", + "last_updated": "2025-12-12T20:12:18.209Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -2558,7 +2588,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "79455c62-cdb1-4f14-bf44-8e76acc020a6" + "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" }, "_type": "Component", "ascending": { @@ -2962,7 +2992,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-12-03T21:41:00.320Z", + "last_updated": "2025-12-12T20:12:18.209Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -3009,7 +3039,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "79455c62-cdb1-4f14-bf44-8e76acc020a6" + "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" }, "_type": "Component", "ascending": { @@ -4126,7 +4156,7 @@ "x": 2261.865622928042, "y": 1349.2821108833643 }, - "selected": true, + "selected": false, "type": "genericNode" }, { @@ -4163,7 +4193,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-12-03T21:41:00.158Z", + "last_updated": "2025-12-12T20:12:18.131Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -4231,7 +4261,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "79455c62-cdb1-4f14-bf44-8e76acc020a6" + "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" }, "_type": "Component", "api_base": { @@ -4688,7 +4718,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-12-03T21:41:00.159Z", + "last_updated": "2025-12-12T20:12:18.132Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -4756,7 +4786,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "79455c62-cdb1-4f14-bf44-8e76acc020a6" + "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" }, "_type": "Component", "api_base": { @@ -4969,8 +4999,7 @@ "load_from_db": false, "name": "model", "options": [ - "embeddinggemma:latest", - "mxbai-embed-large:latest", + "all-minilm:latest", "nomic-embed-text:latest" ], "options_metadata": [], @@ -4986,7 +5015,7 @@ "trace_as_metadata": true, "track_in_telemetry": true, "type": "str", - "value": "embeddinggemma:latest" + "value": "all-minilm:latest" }, "model_kwargs": { "_input_type": "DictInput", @@ -5215,7 +5244,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-12-03T21:41:00.159Z", + "last_updated": "2025-12-12T20:12:18.133Z", "legacy": false, "lf_version": "1.7.0.dev21", "metadata": { @@ -5283,7 +5312,7 @@ "value": "5488df7c-b93f-4f87-a446-b67028bc0813" }, "_frontend_node_folder_id": { - "value": "79455c62-cdb1-4f14-bf44-8e76acc020a6" + "value": "75fd27c1-8f4b-46a1-88bb-a8a8e72719e3" }, "_type": "Component", "api_base": { @@ -5708,15 +5737,16 @@ } ], "viewport": { - "x": -848.3573799283768, - "y": -648.7033245837173, - "zoom": 0.6472397864500404 + "x": 249.3666737262397, + "y": -156.8776378758762, + "zoom": 0.38977017930844676 } }, "description": "Load your data for chat context with Retrieval Augmented Generation.", "endpoint_name": null, "id": "5488df7c-b93f-4f87-a446-b67028bc0813", "is_component": false, + "locked": true, "last_tested_version": "1.7.0.dev21", "name": "OpenSearch Ingestion Flow", "tags": [ @@ -5725,4 +5755,4 @@ "rag", "q-a" ] -} +} \ No newline at end of file diff --git a/flows/openrag_agent.json b/flows/openrag_agent.json index d9475aac..bb1adc71 100644 --- a/flows/openrag_agent.json +++ b/flows/openrag_agent.json @@ -4787,7 +4787,7 @@ "is_component": false, "locked": true, "last_tested_version": "1.7.0.dev21", - "name": "OpenRAG OpenSearch Agent", + "name": "OpenRAG OpenSearch Agent Flow", "tags": [ "assistants", "agents" diff --git a/flows/openrag_nudges.json b/flows/openrag_nudges.json index d9d79e60..475833f9 100644 --- a/flows/openrag_nudges.json +++ b/flows/openrag_nudges.json @@ -4114,7 +4114,7 @@ "is_component": false, "locked": true, "last_tested_version": "1.7.0.dev21", - "name": "OpenRAG OpenSearch Nudges", + "name": "OpenRAG OpenSearch Nudges Flow", "tags": [ "assistants", "agents" diff --git a/flows/openrag_url_mcp.json b/flows/openrag_url_mcp.json index 29d4d12d..26ef5dcb 100644 --- a/flows/openrag_url_mcp.json +++ b/flows/openrag_url_mcp.json @@ -429,6 +429,91 @@ "sourceHandle": "{œdataTypeœ:œEmbeddingModelœ,œidœ:œEmbeddingModel-Rp0iIœ,œnameœ:œembeddingsœ,œoutput_typesœ:[œEmbeddingsœ]}", "target": "OpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGV", "targetHandle": "{œfieldNameœ:œembeddingœ,œidœ:œOpenSearchVectorStoreComponentMultimodalMultiEmbedding-PMGGVœ,œinputTypesœ:[œEmbeddingsœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "URLComponent", + "id": "URLComponent-lnA0q", + "name": "page_results", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "df", + "id": "DataFrameOperations-NpdW5", + "inputTypes": [ + "DataFrame" + ], + "type": "other" + } + }, + "id": "xy-edge__URLComponent-lnA0q{œdataTypeœ:œURLComponentœ,œidœ:œURLComponent-lnA0qœ,œnameœ:œpage_resultsœ,œoutput_typesœ:[œDataFrameœ]}-DataFrameOperations-NpdW5{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-NpdW5œ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}", + "selected": false, + "source": "URLComponent-lnA0q", + "sourceHandle": "{œdataTypeœ:œURLComponentœ,œidœ:œURLComponent-lnA0qœ,œnameœ:œpage_resultsœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "DataFrameOperations-NpdW5", + "targetHandle": "{œfieldNameœ:œdfœ,œidœ:œDataFrameOperations-NpdW5œ,œinputTypesœ:[œDataFrameœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "DataFrameOperations", + "id": "DataFrameOperations-NpdW5", + "name": "output", + "output_types": [ + "DataFrame" + ] + }, + "targetHandle": { + "fieldName": "input_data", + "id": "ParserComponent-1eim1", + "inputTypes": [ + "DataFrame", + "Data" + ], + "type": "other" + } + }, + "id": "xy-edge__DataFrameOperations-NpdW5{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-NpdW5œ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}-ParserComponent-1eim1{œfieldNameœ:œinput_dataœ,œidœ:œParserComponent-1eim1œ,œinputTypesœ:[œDataFrameœ,œDataœ],œtypeœ:œotherœ}", + "selected": false, + "source": "DataFrameOperations-NpdW5", + "sourceHandle": "{œdataTypeœ:œDataFrameOperationsœ,œidœ:œDataFrameOperations-NpdW5œ,œnameœ:œoutputœ,œoutput_typesœ:[œDataFrameœ]}", + "target": "ParserComponent-1eim1", + "targetHandle": "{œfieldNameœ:œinput_dataœ,œidœ:œParserComponent-1eim1œ,œinputTypesœ:[œDataFrameœ,œDataœ],œtypeœ:œotherœ}" + }, + { + "animated": false, + "className": "", + "data": { + "sourceHandle": { + "dataType": "ParserComponent", + "id": "ParserComponent-1eim1", + "name": "parsed_text", + "output_types": [ + "Message" + ] + }, + "targetHandle": { + "fieldName": "new_column_value", + "id": "DataFrameOperations-hqIoy", + "inputTypes": [ + "Message" + ], + "type": "str" + } + }, + "id": "xy-edge__ParserComponent-1eim1{œdataTypeœ:œParserComponentœ,œidœ:œParserComponent-1eim1œ,œnameœ:œparsed_textœ,œoutput_typesœ:[œMessageœ]}-DataFrameOperations-hqIoy{œfieldNameœ:œnew_column_valueœ,œidœ:œDataFrameOperations-hqIoyœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}", + "selected": false, + "source": "ParserComponent-1eim1", + "sourceHandle": "{œdataTypeœ:œParserComponentœ,œidœ:œParserComponent-1eim1œ,œnameœ:œparsed_textœ,œoutput_typesœ:[œMessageœ]}", + "target": "DataFrameOperations-hqIoy", + "targetHandle": "{œfieldNameœ:œnew_column_valueœ,œidœ:œDataFrameOperations-hqIoyœ,œinputTypesœ:[œMessageœ],œtypeœ:œstrœ}" } ], "nodes": [ @@ -706,7 +791,7 @@ "frozen": false, "icon": "layout-template", "legacy": false, - "lf_version": "1.6.0", + "lf_version": "1.7.0.dev21", "metadata": { "code_hash": "4c72ce0f2e34", "dependencies": { @@ -1051,8 +1136,8 @@ "width": 320 }, "position": { - "x": 1253.2399253814751, - "y": 1554.2313683997174 + "x": 417.1342145817257, + "y": 1355.0755805523784 }, "selected": false, "type": "genericNode" @@ -1089,7 +1174,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-11-26T06:11:22.958Z", + "last_updated": "2025-12-12T20:28:32.647Z", "legacy": false, "metadata": { "code_hash": "904f4eaebccd", @@ -1135,7 +1220,7 @@ "value": "72c3d17c-2dac-4a73-b48a-6518473d7830" }, "_frontend_node_folder_id": { - "value": "131daebd-f11a-4072-9e20-1e1f903d01b0" + "value": "2bee9dd9-f030-469f-a568-6fcb3a6e7140" }, "_type": "Component", "ascending": { @@ -1333,7 +1418,7 @@ ], "list": false, "list_add_label": "Add More", - "load_from_db": true, + "load_from_db": false, "name": "new_column_value", "override_skip": false, "placeholder": "", @@ -1345,7 +1430,7 @@ "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "FILENAME" + "value": "" }, "num_rows": { "_input_type": "IntInput", @@ -1501,8 +1586,8 @@ "width": 320 }, "position": { - "x": 1601.8752590736613, - "y": 1442.944202002645 + "x": 1578.348410746631, + "y": 1137.0951737512514 }, "selected": false, "type": "genericNode" @@ -1539,7 +1624,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-11-26T06:11:22.960Z", + "last_updated": "2025-12-12T20:28:32.648Z", "legacy": false, "metadata": { "code_hash": "904f4eaebccd", @@ -1585,7 +1670,7 @@ "value": "72c3d17c-2dac-4a73-b48a-6518473d7830" }, "_frontend_node_folder_id": { - "value": "131daebd-f11a-4072-9e20-1e1f903d01b0" + "value": "2bee9dd9-f030-469f-a568-6fcb3a6e7140" }, "_type": "Component", "ascending": { @@ -1989,7 +2074,7 @@ ], "frozen": false, "icon": "table", - "last_updated": "2025-11-26T06:11:22.961Z", + "last_updated": "2025-12-12T20:28:32.648Z", "legacy": false, "metadata": { "code_hash": "904f4eaebccd", @@ -2035,7 +2120,7 @@ "value": "72c3d17c-2dac-4a73-b48a-6518473d7830" }, "_frontend_node_folder_id": { - "value": "131daebd-f11a-4072-9e20-1e1f903d01b0" + "value": "2bee9dd9-f030-469f-a568-6fcb3a6e7140" }, "_type": "Component", "ascending": { @@ -2435,6 +2520,7 @@ "frozen": false, "icon": "MessagesSquare", "legacy": false, + "lf_version": "1.7.0.dev21", "metadata": { "code_hash": "7a26c54d89ed", "dependencies": { @@ -2698,8 +2784,8 @@ "width": 320 }, "position": { - "x": 850.439672560931, - "y": 1627.238751925257 + "x": 2.9942378786288373, + "y": 1375.037939780366 }, "selected": false, "type": "genericNode" @@ -3042,7 +3128,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-11-26T06:11:22.856Z", + "last_updated": "2025-12-12T20:28:32.527Z", "legacy": false, "metadata": { "code_hash": "0e2d6fe67a26", @@ -3109,12 +3195,12 @@ "value": "72c3d17c-2dac-4a73-b48a-6518473d7830" }, "_frontend_node_folder_id": { - "value": "131daebd-f11a-4072-9e20-1e1f903d01b0" + "value": "2bee9dd9-f030-469f-a568-6fcb3a6e7140" }, "_type": "Component", "api_base": { "_input_type": "MessageTextInput", - "advanced": true, + "advanced": false, "display_name": "API Base URL", "dynamic": false, "info": "Base URL for the API. Leave empty for default.", @@ -3123,7 +3209,7 @@ ], "list": false, "list_add_label": "Add More", - "load_from_db": false, + "load_from_db": true, "name": "api_base", "override_skip": false, "placeholder": "", @@ -3135,12 +3221,12 @@ "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "" + "value": "OLLAMA_BASE_URL" }, "api_key": { "_input_type": "SecretStrInput", "advanced": false, - "display_name": "OpenAI API Key", + "display_name": "API Key (Optional)", "dynamic": false, "info": "Model Provider API key", "input_types": [], @@ -3151,7 +3237,7 @@ "placeholder": "", "real_time_refresh": true, "required": false, - "show": true, + "show": false, "title_case": false, "track_in_telemetry": false, "type": "str", @@ -3320,9 +3406,9 @@ "info": "Select the embedding model to use", "name": "model", "options": [ - "text-embedding-3-small", - "text-embedding-3-large", - "text-embedding-ada-002" + "embeddinggemma:latest", + "mxbai-embed-large:latest", + "nomic-embed-text:latest" ], "options_metadata": [], "override_skip": false, @@ -3337,7 +3423,7 @@ "trace_as_metadata": true, "track_in_telemetry": true, "type": "str", - "value": "text-embedding-3-small" + "value": "embeddinggemma:latest" }, "model_kwargs": { "_input_type": "DictInput", @@ -3370,20 +3456,20 @@ ], "list": false, "list_add_label": "Add More", - "load_from_db": false, + "load_from_db": true, "name": "ollama_base_url", "override_skip": false, "placeholder": "", "real_time_refresh": true, "required": false, - "show": false, + "show": true, "title_case": false, "tool_mode": false, "trace_as_input": true, "trace_as_metadata": true, "track_in_telemetry": false, "type": "str", - "value": "" + "value": "OLLAMA_BASE_URL" }, "project_id": { "_input_type": "MessageTextInput", @@ -3447,7 +3533,7 @@ "trace_as_metadata": true, "track_in_telemetry": true, "type": "str", - "value": "OpenAI" + "value": "Ollama" }, "request_timeout": { "_input_type": "FloatInput", @@ -3518,7 +3604,7 @@ "dragging": false, "id": "EmbeddingModel-XjV5v", "measured": { - "height": 369, + "height": 451, "width": 320 }, "position": { @@ -4270,7 +4356,7 @@ ], "frozen": false, "icon": "braces", - "last_updated": "2025-11-26T06:11:22.857Z", + "last_updated": "2025-12-12T20:28:32.528Z", "legacy": false, "lf_version": "1.6.3.dev0", "metadata": {}, @@ -4320,7 +4406,7 @@ "value": "72c3d17c-2dac-4a73-b48a-6518473d7830" }, "_frontend_node_folder_id": { - "value": "131daebd-f11a-4072-9e20-1e1f903d01b0" + "value": "2bee9dd9-f030-469f-a568-6fcb3a6e7140" }, "_type": "Component", "code": { @@ -5025,7 +5111,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-11-26T06:11:22.858Z", + "last_updated": "2025-12-12T20:28:32.529Z", "legacy": false, "metadata": { "code_hash": "0e2d6fe67a26", @@ -5092,7 +5178,7 @@ "value": "72c3d17c-2dac-4a73-b48a-6518473d7830" }, "_frontend_node_folder_id": { - "value": "131daebd-f11a-4072-9e20-1e1f903d01b0" + "value": "2bee9dd9-f030-469f-a568-6fcb3a6e7140" }, "_type": "Component", "api_base": { @@ -5303,7 +5389,11 @@ "info": "Select the embedding model to use", "load_from_db": false, "name": "model", - "options": [], + "options": [ + "embeddinggemma:latest", + "mxbai-embed-large:latest", + "nomic-embed-text:latest" + ], "options_metadata": [], "override_skip": false, "placeholder": "", @@ -5317,7 +5407,7 @@ "trace_as_metadata": true, "track_in_telemetry": true, "type": "str", - "value": "" + "value": "embeddinggemma:latest" }, "model_kwargs": { "_input_type": "DictInput", @@ -5548,7 +5638,7 @@ ], "frozen": false, "icon": "binary", - "last_updated": "2025-11-26T06:11:22.860Z", + "last_updated": "2025-12-12T20:28:32.529Z", "legacy": false, "metadata": { "code_hash": "0e2d6fe67a26", @@ -5615,7 +5705,7 @@ "value": "72c3d17c-2dac-4a73-b48a-6518473d7830" }, "_frontend_node_folder_id": { - "value": "131daebd-f11a-4072-9e20-1e1f903d01b0" + "value": "2bee9dd9-f030-469f-a568-6fcb3a6e7140" }, "_type": "Component", "api_base": { @@ -6040,21 +6130,668 @@ }, "selected": false, "type": "genericNode" + }, + { + "data": { + "description": "Perform various operations on a DataFrame.", + "display_name": "DataFrame Operations", + "id": "DataFrameOperations-NpdW5", + "node": { + "base_classes": [ + "DataFrame" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Perform various operations on a DataFrame.", + "display_name": "DataFrame Operations", + "documentation": "https://docs.langflow.org/dataframe-operations", + "edited": false, + "field_order": [ + "df", + "operation", + "column_name", + "filter_value", + "filter_operator", + "ascending", + "new_column_name", + "new_column_value", + "columns_to_select", + "num_rows", + "replace_value", + "replacement_value" + ], + "frozen": false, + "icon": "table", + "last_updated": "2025-12-12T20:28:32.649Z", + "legacy": false, + "lf_version": "1.7.0.dev21", + "metadata": { + "code_hash": "904f4eaebccd", + "dependencies": { + "dependencies": [ + { + "name": "pandas", + "version": "2.2.3" + }, + { + "name": "lfx", + "version": null + } + ], + "total_dependencies": 2 + }, + "module": "custom_components.dataframe_operations" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "DataFrame", + "group_outputs": false, + "loop_types": null, + "method": "perform_operation", + "name": "output", + "options": null, + "required_inputs": null, + "selected": "DataFrame", + "tool_mode": true, + "types": [ + "DataFrame" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_frontend_node_flow_id": { + "value": "72c3d17c-2dac-4a73-b48a-6518473d7830" + }, + "_frontend_node_folder_id": { + "value": "2bee9dd9-f030-469f-a568-6fcb3a6e7140" + }, + "_type": "Component", + "ascending": { + "_input_type": "BoolInput", + "advanced": false, + "display_name": "Sort Ascending", + "dynamic": true, + "info": "Whether to sort in ascending order.", + "list": false, + "list_add_label": "Add More", + "name": "ascending", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "bool", + "value": true + }, + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "import pandas as pd\n\nfrom lfx.custom.custom_component.component import Component\nfrom lfx.inputs import SortableListInput\nfrom lfx.io import BoolInput, DataFrameInput, DropdownInput, IntInput, MessageTextInput, Output, StrInput\nfrom lfx.log.logger import logger\nfrom lfx.schema.dataframe import DataFrame\n\n\nclass DataFrameOperationsComponent(Component):\n display_name = \"DataFrame Operations\"\n description = \"Perform various operations on a DataFrame.\"\n documentation: str = \"https://docs.langflow.org/dataframe-operations\"\n icon = \"table\"\n name = \"DataFrameOperations\"\n\n OPERATION_CHOICES = [\n \"Add Column\",\n \"Drop Column\",\n \"Filter\",\n \"Head\",\n \"Rename Column\",\n \"Replace Value\",\n \"Select Columns\",\n \"Sort\",\n \"Tail\",\n \"Drop Duplicates\",\n ]\n\n inputs = [\n DataFrameInput(\n name=\"df\",\n display_name=\"DataFrame\",\n info=\"The input DataFrame to operate on.\",\n required=True,\n ),\n SortableListInput(\n name=\"operation\",\n display_name=\"Operation\",\n placeholder=\"Select Operation\",\n info=\"Select the DataFrame operation to perform.\",\n options=[\n {\"name\": \"Add Column\", \"icon\": \"plus\"},\n {\"name\": \"Drop Column\", \"icon\": \"minus\"},\n {\"name\": \"Filter\", \"icon\": \"filter\"},\n {\"name\": \"Head\", \"icon\": \"arrow-up\"},\n {\"name\": \"Rename Column\", \"icon\": \"pencil\"},\n {\"name\": \"Replace Value\", \"icon\": \"replace\"},\n {\"name\": \"Select Columns\", \"icon\": \"columns\"},\n {\"name\": \"Sort\", \"icon\": \"arrow-up-down\"},\n {\"name\": \"Tail\", \"icon\": \"arrow-down\"},\n {\"name\": \"Drop Duplicates\", \"icon\": \"copy-x\"},\n ],\n real_time_refresh=True,\n limit=1,\n ),\n StrInput(\n name=\"column_name\",\n display_name=\"Column Name\",\n info=\"The column name to use for the operation.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"filter_value\",\n display_name=\"Filter Value\",\n info=\"The value to filter rows by.\",\n dynamic=True,\n show=False,\n ),\n DropdownInput(\n name=\"filter_operator\",\n display_name=\"Filter Operator\",\n options=[\n \"equals\",\n \"not equals\",\n \"contains\",\n \"not contains\",\n \"starts with\",\n \"ends with\",\n \"greater than\",\n \"less than\",\n ],\n value=\"equals\",\n info=\"The operator to apply for filtering rows.\",\n advanced=False,\n dynamic=True,\n show=False,\n ),\n BoolInput(\n name=\"ascending\",\n display_name=\"Sort Ascending\",\n info=\"Whether to sort in ascending order.\",\n dynamic=True,\n show=False,\n value=True,\n ),\n StrInput(\n name=\"new_column_name\",\n display_name=\"New Column Name\",\n info=\"The new column name when renaming or adding a column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"new_column_value\",\n display_name=\"New Column Value\",\n info=\"The value to populate the new column with.\",\n dynamic=True,\n show=False,\n ),\n StrInput(\n name=\"columns_to_select\",\n display_name=\"Columns to Select\",\n dynamic=True,\n is_list=True,\n show=False,\n ),\n IntInput(\n name=\"num_rows\",\n display_name=\"Number of Rows\",\n info=\"Number of rows to return (for head/tail).\",\n dynamic=True,\n show=False,\n value=5,\n ),\n MessageTextInput(\n name=\"replace_value\",\n display_name=\"Value to Replace\",\n info=\"The value to replace in the column.\",\n dynamic=True,\n show=False,\n ),\n MessageTextInput(\n name=\"replacement_value\",\n display_name=\"Replacement Value\",\n info=\"The value to replace with.\",\n dynamic=True,\n show=False,\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"DataFrame\",\n name=\"output\",\n method=\"perform_operation\",\n info=\"The resulting DataFrame after the operation.\",\n )\n ]\n\n def update_build_config(self, build_config, field_value, field_name=None):\n dynamic_fields = [\n \"column_name\",\n \"filter_value\",\n \"filter_operator\",\n \"ascending\",\n \"new_column_name\",\n \"new_column_value\",\n \"columns_to_select\",\n \"num_rows\",\n \"replace_value\",\n \"replacement_value\",\n ]\n for field in dynamic_fields:\n build_config[field][\"show\"] = False\n\n if field_name == \"operation\":\n # Handle SortableListInput format\n if isinstance(field_value, list):\n operation_name = field_value[0].get(\"name\", \"\") if field_value else \"\"\n else:\n operation_name = field_value or \"\"\n\n # If no operation selected, all dynamic fields stay hidden (already set to False above)\n if not operation_name:\n return build_config\n\n if operation_name == \"Filter\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"filter_value\"][\"show\"] = True\n build_config[\"filter_operator\"][\"show\"] = True\n elif operation_name == \"Sort\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"ascending\"][\"show\"] = True\n elif operation_name == \"Drop Column\":\n build_config[\"column_name\"][\"show\"] = True\n elif operation_name == \"Rename Column\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"new_column_name\"][\"show\"] = True\n elif operation_name == \"Add Column\":\n build_config[\"new_column_name\"][\"show\"] = True\n build_config[\"new_column_value\"][\"show\"] = True\n elif operation_name == \"Select Columns\":\n build_config[\"columns_to_select\"][\"show\"] = True\n elif operation_name in {\"Head\", \"Tail\"}:\n build_config[\"num_rows\"][\"show\"] = True\n elif operation_name == \"Replace Value\":\n build_config[\"column_name\"][\"show\"] = True\n build_config[\"replace_value\"][\"show\"] = True\n build_config[\"replacement_value\"][\"show\"] = True\n elif operation_name == \"Drop Duplicates\":\n build_config[\"column_name\"][\"show\"] = True\n\n return build_config\n\n def perform_operation(self) -> DataFrame:\n df_copy = self.df.copy()\n\n # Handle SortableListInput format for operation\n operation_input = getattr(self, \"operation\", [])\n if isinstance(operation_input, list) and len(operation_input) > 0:\n op = operation_input[0].get(\"name\", \"\")\n else:\n op = \"\"\n\n # If no operation selected, return original DataFrame\n if not op:\n return df_copy\n\n if op == \"Filter\":\n return self.filter_rows_by_value(df_copy)\n if op == \"Sort\":\n return self.sort_by_column(df_copy)\n if op == \"Drop Column\":\n return self.drop_column(df_copy)\n if op == \"Rename Column\":\n return self.rename_column(df_copy)\n if op == \"Add Column\":\n return self.add_column(df_copy)\n if op == \"Select Columns\":\n return self.select_columns(df_copy)\n if op == \"Head\":\n return self.head(df_copy)\n if op == \"Tail\":\n return self.tail(df_copy)\n if op == \"Replace Value\":\n return self.replace_values(df_copy)\n if op == \"Drop Duplicates\":\n return self.drop_duplicates(df_copy)\n msg = f\"Unsupported operation: {op}\"\n logger.error(msg)\n raise ValueError(msg)\n\n def filter_rows_by_value(self, df: DataFrame) -> DataFrame:\n column = df[self.column_name]\n filter_value = self.filter_value\n\n # Handle regular DropdownInput format (just a string value)\n operator = getattr(self, \"filter_operator\", \"equals\") # Default to equals for backward compatibility\n\n if operator == \"equals\":\n mask = column == filter_value\n elif operator == \"not equals\":\n mask = column != filter_value\n elif operator == \"contains\":\n mask = column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"not contains\":\n mask = ~column.astype(str).str.contains(str(filter_value), na=False)\n elif operator == \"starts with\":\n mask = column.astype(str).str.startswith(str(filter_value), na=False)\n elif operator == \"ends with\":\n mask = column.astype(str).str.endswith(str(filter_value), na=False)\n elif operator == \"greater than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column > numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) > str(filter_value)\n elif operator == \"less than\":\n try:\n # Try to convert filter_value to numeric for comparison\n numeric_value = pd.to_numeric(filter_value)\n mask = column < numeric_value\n except (ValueError, TypeError):\n # If conversion fails, compare as strings\n mask = column.astype(str) < str(filter_value)\n else:\n mask = column == filter_value # Fallback to equals\n\n return DataFrame(df[mask])\n\n def sort_by_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.sort_values(by=self.column_name, ascending=self.ascending))\n\n def drop_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop(columns=[self.column_name]))\n\n def rename_column(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.rename(columns={self.column_name: self.new_column_name}))\n\n def add_column(self, df: DataFrame) -> DataFrame:\n df[self.new_column_name] = [self.new_column_value] * len(df)\n return DataFrame(df)\n\n def select_columns(self, df: DataFrame) -> DataFrame:\n columns = [col.strip() for col in self.columns_to_select]\n return DataFrame(df[columns])\n\n def head(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.head(self.num_rows))\n\n def tail(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.tail(self.num_rows))\n\n def replace_values(self, df: DataFrame) -> DataFrame:\n df[self.column_name] = df[self.column_name].replace(self.replace_value, self.replacement_value)\n return DataFrame(df)\n\n def drop_duplicates(self, df: DataFrame) -> DataFrame:\n return DataFrame(df.drop_duplicates(subset=self.column_name))\n" + }, + "column_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Column Name", + "dynamic": true, + "info": "The column name to use for the operation.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "column_name", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + }, + "columns_to_select": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "Columns to Select", + "dynamic": true, + "info": "", + "list": true, + "list_add_label": "Add More", + "load_from_db": false, + "name": "columns_to_select", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": [ + "title" + ] + }, + "df": { + "_input_type": "DataFrameInput", + "advanced": false, + "display_name": "DataFrame", + "dynamic": false, + "info": "The input DataFrame to operate on.", + "input_types": [ + "DataFrame" + ], + "list": false, + "list_add_label": "Add More", + "name": "df", + "override_skip": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "other", + "value": "" + }, + "filter_operator": { + "_input_type": "DropdownInput", + "advanced": false, + "combobox": false, + "dialog_inputs": {}, + "display_name": "Filter Operator", + "dynamic": true, + "external_options": {}, + "info": "The operator to apply for filtering rows.", + "name": "filter_operator", + "options": [ + "equals", + "not equals", + "contains", + "not contains", + "starts with", + "ends with", + "greater than", + "less than" + ], + "options_metadata": [], + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "toggle": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "str", + "value": "equals" + }, + "filter_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Filter Value", + "dynamic": true, + "info": "The value to filter rows by.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "filter_value", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + }, + "is_refresh": false, + "new_column_name": { + "_input_type": "StrInput", + "advanced": false, + "display_name": "New Column Name", + "dynamic": true, + "info": "The new column name when renaming or adding a column.", + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "new_column_name", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "filename" + }, + "new_column_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "New Column Value", + "dynamic": true, + "info": "The value to populate the new column with.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "new_column_value", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + }, + "num_rows": { + "_input_type": "IntInput", + "advanced": false, + "display_name": "Number of Rows", + "dynamic": true, + "info": "Number of rows to return (for head/tail).", + "list": false, + "list_add_label": "Add More", + "name": "num_rows", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "int", + "value": 5 + }, + "operation": { + "_input_type": "SortableListInput", + "advanced": false, + "display_name": "Operation", + "dynamic": false, + "info": "Select the DataFrame operation to perform.", + "limit": 1, + "load_from_db": false, + "name": "operation", + "options": [ + { + "icon": "plus", + "name": "Add Column" + }, + { + "icon": "minus", + "name": "Drop Column" + }, + { + "icon": "filter", + "name": "Filter" + }, + { + "icon": "arrow-up", + "name": "Head" + }, + { + "icon": "pencil", + "name": "Rename Column" + }, + { + "icon": "replace", + "name": "Replace Value" + }, + { + "icon": "columns", + "name": "Select Columns" + }, + { + "icon": "arrow-up-down", + "name": "Sort" + }, + { + "icon": "arrow-down", + "name": "Tail" + }, + { + "icon": "copy-x", + "name": "Drop Duplicates" + } + ], + "override_skip": false, + "placeholder": "Select Operation", + "real_time_refresh": true, + "required": false, + "search_category": [], + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "sortableList", + "value": [ + { + "chosen": false, + "icon": "columns", + "name": "Select Columns", + "selected": false + } + ] + }, + "replace_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Value to Replace", + "dynamic": true, + "info": "The value to replace in the column.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "replace_value", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + }, + "replacement_value": { + "_input_type": "MessageTextInput", + "advanced": false, + "display_name": "Replacement Value", + "dynamic": true, + "info": "The value to replace with.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "replacement_value", + "override_skip": false, + "placeholder": "", + "required": false, + "show": false, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "" + } + }, + "tool_mode": false + }, + "showNode": true, + "type": "DataFrameOperations" + }, + "dragging": false, + "id": "DataFrameOperations-NpdW5", + "measured": { + "height": 317, + "width": 320 + }, + "position": { + "x": 856.1961817994918, + "y": 1687.1833235248055 + }, + "selected": false, + "type": "genericNode" + }, + { + "data": { + "id": "ParserComponent-1eim1", + "node": { + "base_classes": [ + "Message" + ], + "beta": false, + "conditional_paths": [], + "custom_fields": {}, + "description": "Extracts text using a template.", + "display_name": "Parser", + "documentation": "https://docs.langflow.org/parser", + "edited": false, + "field_order": [ + "input_data", + "mode", + "pattern", + "sep" + ], + "frozen": false, + "icon": "braces", + "legacy": false, + "lf_version": "1.7.0.dev21", + "metadata": { + "code_hash": "3cda25c3f7b5", + "dependencies": { + "dependencies": [ + { + "name": "lfx", + "version": "0.2.0.dev21" + } + ], + "total_dependencies": 1 + }, + "module": "lfx.components.processing.parser.ParserComponent" + }, + "minimized": false, + "output_types": [], + "outputs": [ + { + "allows_loop": false, + "cache": true, + "display_name": "Parsed Text", + "group_outputs": false, + "method": "parse_combined_text", + "name": "parsed_text", + "selected": "Message", + "tool_mode": true, + "types": [ + "Message" + ], + "value": "__UNDEFINED__" + } + ], + "pinned": false, + "template": { + "_type": "Component", + "code": { + "advanced": true, + "dynamic": true, + "fileTypes": [], + "file_path": "", + "info": "", + "list": false, + "load_from_db": false, + "multiline": true, + "name": "code", + "password": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "type": "code", + "value": "from lfx.custom.custom_component.component import Component\nfrom lfx.helpers.data import safe_convert\nfrom lfx.inputs.inputs import BoolInput, HandleInput, MessageTextInput, MultilineInput, TabInput\nfrom lfx.schema.data import Data\nfrom lfx.schema.dataframe import DataFrame\nfrom lfx.schema.message import Message\nfrom lfx.template.field.base import Output\n\n\nclass ParserComponent(Component):\n display_name = \"Parser\"\n description = \"Extracts text using a template.\"\n documentation: str = \"https://docs.langflow.org/parser\"\n icon = \"braces\"\n\n inputs = [\n HandleInput(\n name=\"input_data\",\n display_name=\"Data or DataFrame\",\n input_types=[\"DataFrame\", \"Data\"],\n info=\"Accepts either a DataFrame or a Data object.\",\n required=True,\n ),\n TabInput(\n name=\"mode\",\n display_name=\"Mode\",\n options=[\"Parser\", \"Stringify\"],\n value=\"Parser\",\n info=\"Convert into raw string instead of using a template.\",\n real_time_refresh=True,\n ),\n MultilineInput(\n name=\"pattern\",\n display_name=\"Template\",\n info=(\n \"Use variables within curly brackets to extract column values for DataFrames \"\n \"or key values for Data.\"\n \"For example: `Name: {Name}, Age: {Age}, Country: {Country}`\"\n ),\n value=\"Text: {text}\", # Example default\n dynamic=True,\n show=True,\n required=True,\n ),\n MessageTextInput(\n name=\"sep\",\n display_name=\"Separator\",\n advanced=True,\n value=\"\\n\",\n info=\"String used to separate rows/items.\",\n ),\n ]\n\n outputs = [\n Output(\n display_name=\"Parsed Text\",\n name=\"parsed_text\",\n info=\"Formatted text output.\",\n method=\"parse_combined_text\",\n ),\n ]\n\n def update_build_config(self, build_config, field_value, field_name=None):\n \"\"\"Dynamically hide/show `template` and enforce requirement based on `stringify`.\"\"\"\n if field_name == \"mode\":\n build_config[\"pattern\"][\"show\"] = self.mode == \"Parser\"\n build_config[\"pattern\"][\"required\"] = self.mode == \"Parser\"\n if field_value:\n clean_data = BoolInput(\n name=\"clean_data\",\n display_name=\"Clean Data\",\n info=(\n \"Enable to clean the data by removing empty rows and lines \"\n \"in each cell of the DataFrame/ Data object.\"\n ),\n value=True,\n advanced=True,\n required=False,\n )\n build_config[\"clean_data\"] = clean_data.to_dict()\n else:\n build_config.pop(\"clean_data\", None)\n\n return build_config\n\n def _clean_args(self):\n \"\"\"Prepare arguments based on input type.\"\"\"\n input_data = self.input_data\n\n match input_data:\n case list() if all(isinstance(item, Data) for item in input_data):\n msg = \"List of Data objects is not supported.\"\n raise ValueError(msg)\n case DataFrame():\n return input_data, None\n case Data():\n return None, input_data\n case dict() if \"data\" in input_data:\n try:\n if \"columns\" in input_data: # Likely a DataFrame\n return DataFrame.from_dict(input_data), None\n # Likely a Data object\n return None, Data(**input_data)\n except (TypeError, ValueError, KeyError) as e:\n msg = f\"Invalid structured input provided: {e!s}\"\n raise ValueError(msg) from e\n case _:\n msg = f\"Unsupported input type: {type(input_data)}. Expected DataFrame or Data.\"\n raise ValueError(msg)\n\n def parse_combined_text(self) -> Message:\n \"\"\"Parse all rows/items into a single text or convert input to string if `stringify` is enabled.\"\"\"\n # Early return for stringify option\n if self.mode == \"Stringify\":\n return self.convert_to_string()\n\n df, data = self._clean_args()\n\n lines = []\n if df is not None:\n for _, row in df.iterrows():\n formatted_text = self.pattern.format(**row.to_dict())\n lines.append(formatted_text)\n elif data is not None:\n # Use format_map with a dict that returns default_value for missing keys\n class DefaultDict(dict):\n def __missing__(self, key):\n return data.default_value or \"\"\n\n formatted_text = self.pattern.format_map(DefaultDict(data.data))\n lines.append(formatted_text)\n\n combined_text = self.sep.join(lines)\n self.status = combined_text\n return Message(text=combined_text)\n\n def convert_to_string(self) -> Message:\n \"\"\"Convert input data to string with proper error handling.\"\"\"\n result = \"\"\n if isinstance(self.input_data, list):\n result = \"\\n\".join([safe_convert(item, clean_data=self.clean_data or False) for item in self.input_data])\n else:\n result = safe_convert(self.input_data or False)\n self.log(f\"Converted to string with length: {len(result)}\")\n\n message = Message(text=result)\n self.status = message\n return message\n" + }, + "input_data": { + "_input_type": "HandleInput", + "advanced": false, + "display_name": "Data or DataFrame", + "dynamic": false, + "info": "Accepts either a DataFrame or a Data object.", + "input_types": [ + "DataFrame", + "Data" + ], + "list": false, + "list_add_label": "Add More", + "name": "input_data", + "override_skip": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "other", + "value": "" + }, + "mode": { + "_input_type": "TabInput", + "advanced": false, + "display_name": "Mode", + "dynamic": false, + "info": "Convert into raw string instead of using a template.", + "name": "mode", + "options": [ + "Parser", + "Stringify" + ], + "override_skip": false, + "placeholder": "", + "real_time_refresh": true, + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_metadata": true, + "track_in_telemetry": true, + "type": "tab", + "value": "Parser" + }, + "pattern": { + "_input_type": "MultilineInput", + "advanced": false, + "ai_enabled": false, + "copy_field": false, + "display_name": "Template", + "dynamic": true, + "info": "Use variables within curly brackets to extract column values for DataFrames or key values for Data.For example: `Name: {Name}, Age: {Age}, Country: {Country}`", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "multiline": true, + "name": "pattern", + "override_skip": false, + "placeholder": "", + "required": true, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "{title}" + }, + "sep": { + "_input_type": "MessageTextInput", + "advanced": true, + "display_name": "Separator", + "dynamic": false, + "info": "String used to separate rows/items.", + "input_types": [ + "Message" + ], + "list": false, + "list_add_label": "Add More", + "load_from_db": false, + "name": "sep", + "override_skip": false, + "placeholder": "", + "required": false, + "show": true, + "title_case": false, + "tool_mode": false, + "trace_as_input": true, + "trace_as_metadata": true, + "track_in_telemetry": false, + "type": "str", + "value": "\n" + } + }, + "tool_mode": false + }, + "showNode": true, + "type": "ParserComponent" + }, + "dragging": false, + "id": "ParserComponent-1eim1", + "measured": { + "height": 329, + "width": 320 + }, + "position": { + "x": 1205.2726296612543, + "y": 1642.3744133698385 + }, + "selected": false, + "type": "genericNode" } ], "viewport": { - "x": -907.3774606121137, - "y": -579.4662259269908, - "zoom": 0.6003073886978839 + "x": 294.64772127588185, + "y": -396.0401212124092, + "zoom": 0.5090092700849728 } }, "description": "This flow is to ingest the URL to open search.", "endpoint_name": null, "id": "72c3d17c-2dac-4a73-b48a-6518473d7830", "is_component": false, - "locked": true, - "mcp_enabled": true, "last_tested_version": "1.7.0.dev21", + "mcp_enabled": true, + "locked": true, "name": "OpenSearch URL Ingestion Flow", "tags": [ "openai", @@ -6062,4 +6799,4 @@ "rag", "q-a" ] -} +} \ No newline at end of file diff --git a/frontend/app/api/queries/useGetAllFiltersQuery.ts b/frontend/app/api/queries/useGetAllFiltersQuery.ts new file mode 100644 index 00000000..5264d981 --- /dev/null +++ b/frontend/app/api/queries/useGetAllFiltersQuery.ts @@ -0,0 +1,36 @@ +import { + type UseQueryOptions, + useQuery, + useQueryClient, +} from "@tanstack/react-query"; +import type { KnowledgeFilter } from "./useGetFiltersSearchQuery"; + +export const useGetAllFiltersQuery = ( + options?: Omit, "queryKey" | "queryFn">, +) => { + const queryClient = useQueryClient(); + + async function getAllFilters(): Promise { + const response = await fetch("/api/knowledge-filter/search", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ query: "", limit: 1000 }), // Fetch all filters + }); + + const json = await response.json(); + if (!response.ok || !json.success) { + // ensure we always return a KnowledgeFilter[] to satisfy the return type + return []; + } + return (json.filters || []) as KnowledgeFilter[]; + } + + return useQuery( + { + queryKey: ["knowledge-filters", "all"], + queryFn: getAllFilters, + ...options, + }, + queryClient, + ); +}; diff --git a/frontend/app/chat/_components/chat-input.tsx b/frontend/app/chat/_components/chat-input.tsx index 29b081c5..cd343eda 100644 --- a/frontend/app/chat/_components/chat-input.tsx +++ b/frontend/app/chat/_components/chat-input.tsx @@ -1,11 +1,12 @@ +import Fuse from "fuse.js"; import { ArrowRight, Check, Funnel, Loader2, Plus } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; import { - forwardRef, - useImperativeHandle, - useMemo, - useRef, - useState, + forwardRef, + useImperativeHandle, + useMemo, + useRef, + useState, } from "react"; import { useDropzone } from "react-dropzone"; import TextareaAutosize from "react-textarea-autosize"; @@ -13,585 +14,593 @@ import { toast } from "sonner"; import type { FilterColor } from "@/components/filter-icon-popover"; import { Button } from "@/components/ui/button"; import { - Popover, - PopoverAnchor, - PopoverContent, + Popover, + PopoverAnchor, + PopoverContent, } from "@/components/ui/popover"; import { useFileDrag } from "@/hooks/use-file-drag"; import { cn } from "@/lib/utils"; -import { useGetFiltersSearchQuery } from "../../api/queries/useGetFiltersSearchQuery"; +import { useGetAllFiltersQuery } from "../../api/queries/useGetAllFiltersQuery"; import type { KnowledgeFilterData } from "../_types/types"; import { FilePreview } from "./file-preview"; import { SelectedKnowledgeFilter } from "./selected-knowledge-filter"; export interface ChatInputHandle { - focusInput: () => void; - clickFileInput: () => void; + focusInput: () => void; + clickFileInput: () => void; } interface ChatInputProps { - input: string; - loading: boolean; - isUploading: boolean; - selectedFilter: KnowledgeFilterData | null; - parsedFilterData: { color?: FilterColor } | null; - uploadedFile: File | null; - onSubmit: (e: React.FormEvent) => void; - onChange: (value: string) => void; - onKeyDown: (e: React.KeyboardEvent) => void; - onFilterSelect: (filter: KnowledgeFilterData | null) => void; - onFilePickerClick: () => void; - setSelectedFilter: (filter: KnowledgeFilterData | null) => void; - setIsFilterHighlighted: (highlighted: boolean) => void; - onFileSelected: (file: File | null) => void; + input: string; + loading: boolean; + isUploading: boolean; + selectedFilter: KnowledgeFilterData | null; + parsedFilterData: { color?: FilterColor } | null; + uploadedFile: File | null; + onSubmit: (e: React.FormEvent) => void; + onChange: (value: string) => void; + onKeyDown: (e: React.KeyboardEvent) => void; + onFilterSelect: (filter: KnowledgeFilterData | null) => void; + onFilePickerClick: () => void; + setSelectedFilter: (filter: KnowledgeFilterData | null) => void; + setIsFilterHighlighted: (highlighted: boolean) => void; + onFileSelected: (file: File | null) => void; } export const ChatInput = forwardRef( - ( - { - input, - loading, - isUploading, - selectedFilter, - parsedFilterData, - uploadedFile, - onSubmit, - onChange, - onKeyDown, - onFilterSelect, - onFilePickerClick, - setSelectedFilter, - setIsFilterHighlighted, - onFileSelected, - }, - ref, - ) => { - const inputRef = useRef(null); - const fileInputRef = useRef(null); - const [textareaHeight, setTextareaHeight] = useState(0); - const isDragging = useFileDrag(); + ( + { + input, + loading, + isUploading, + selectedFilter, + parsedFilterData, + uploadedFile, + onSubmit, + onChange, + onKeyDown, + onFilterSelect, + onFilePickerClick, + setSelectedFilter, + setIsFilterHighlighted, + onFileSelected, + }, + ref, + ) => { + const inputRef = useRef(null); + const fileInputRef = useRef(null); + const [textareaHeight, setTextareaHeight] = useState(0); + const isDragging = useFileDrag(); - // Internal state for filter dropdown - const [isFilterDropdownOpen, setIsFilterDropdownOpen] = useState(false); - const [filterSearchTerm, setFilterSearchTerm] = useState(""); - const [selectedFilterIndex, setSelectedFilterIndex] = useState(0); - const [anchorPosition, setAnchorPosition] = useState<{ - x: number; - y: number; - } | null>(null); + // Internal state for filter dropdown + const [isFilterDropdownOpen, setIsFilterDropdownOpen] = useState(false); + const [filterSearchTerm, setFilterSearchTerm] = useState(""); + const [selectedFilterIndex, setSelectedFilterIndex] = useState(0); + const [anchorPosition, setAnchorPosition] = useState<{ + x: number; + y: number; + } | null>(null); - // Fetch filters using the query hook - const { data: availableFilters = [] } = useGetFiltersSearchQuery( - filterSearchTerm, - 20, - { enabled: isFilterDropdownOpen }, - ); + // Fetch all filters once when dropdown opens + const { data: allFilters = [] } = useGetAllFiltersQuery({ + enabled: isFilterDropdownOpen, + }); - // Filter available filters based on search term - const filteredFilters = useMemo(() => { - return availableFilters.filter((filter) => - filter.name.toLowerCase().includes(filterSearchTerm.toLowerCase()), - ); - }, [availableFilters, filterSearchTerm]); + // Use fuse.js for fuzzy search on client side + const filteredFilters = useMemo(() => { + if (!filterSearchTerm) { + return allFilters.slice(0, 20); // Return first 20 when no search term + } - const { getRootProps, getInputProps } = useDropzone({ - accept: { - "application/pdf": [".pdf"], - "application/msword": [".doc"], - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": - [".docx"], - "text/markdown": [".md"], - }, - maxFiles: 1, - disabled: !isDragging, - onDrop: (acceptedFiles, fileRejections) => { - if (fileRejections.length > 0) { - const message = fileRejections.at(0)?.errors.at(0)?.message; - toast.error(message || "Failed to upload file"); - return; - } - onFileSelected(acceptedFiles[0]); - }, - }); + const fuse = new Fuse(allFilters, { + keys: ["name", "description"], + threshold: 0.3, // 0.0 = perfect match, 1.0 = match anything + includeScore: true, + minMatchCharLength: 1, + }); - useImperativeHandle(ref, () => ({ - focusInput: () => { - inputRef.current?.focus(); - }, - clickFileInput: () => { - fileInputRef.current?.click(); - }, - })); + const results = fuse.search(filterSearchTerm); + return results.map((result) => result.item).slice(0, 20); + }, [allFilters, filterSearchTerm]); - const handleFilePickerChange = (e: React.ChangeEvent) => { - const files = e.target.files; - if (files && files.length > 0) { - onFileSelected(files[0]); - } else { - onFileSelected(null); - } - }; + const { getRootProps, getInputProps } = useDropzone({ + accept: { + "application/pdf": [".pdf"], + "application/msword": [".doc"], + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": + [".docx"], + "text/markdown": [".md"], + }, + maxFiles: 1, + disabled: !isDragging, + onDrop: (acceptedFiles, fileRejections) => { + if (fileRejections.length > 0) { + const message = fileRejections.at(0)?.errors.at(0)?.message; + toast.error(message || "Failed to upload file"); + return; + } + onFileSelected(acceptedFiles[0]); + }, + }); - const onAtClick = () => { - if (!isFilterDropdownOpen) { - setIsFilterDropdownOpen(true); - setFilterSearchTerm(""); - setSelectedFilterIndex(0); + useImperativeHandle(ref, () => ({ + focusInput: () => { + inputRef.current?.focus(); + }, + clickFileInput: () => { + fileInputRef.current?.click(); + }, + })); - // Get button position for popover anchoring - const button = document.querySelector( - "[data-filter-button]", - ) as HTMLElement; - if (button) { - const rect = button.getBoundingClientRect(); - setAnchorPosition({ - x: rect.left + rect.width / 2, - y: rect.top + rect.height / 2 - 12, - }); - } - } else { - setIsFilterDropdownOpen(false); - setAnchorPosition(null); - } - }; + const handleFilePickerChange = (e: React.ChangeEvent) => { + const files = e.target.files; + if (files && files.length > 0) { + onFileSelected(files[0]); + } else { + onFileSelected(null); + } + }; - const handleFilterSelect = (filter: KnowledgeFilterData | null) => { - onFilterSelect(filter); + const onAtClick = () => { + if (!isFilterDropdownOpen) { + setIsFilterDropdownOpen(true); + setFilterSearchTerm(""); + setSelectedFilterIndex(0); - // Remove the @searchTerm from the input - const words = input.split(" "); - const lastWord = words[words.length - 1]; + // Get button position for popover anchoring + const button = document.querySelector( + "[data-filter-button]", + ) as HTMLElement; + if (button) { + const rect = button.getBoundingClientRect(); + setAnchorPosition({ + x: rect.left + rect.width / 2, + y: rect.top + rect.height / 2 - 12, + }); + } + } else { + setIsFilterDropdownOpen(false); + setAnchorPosition(null); + } + }; - if (lastWord.startsWith("@")) { - // Remove the @search term - words.pop(); - onChange(words.join(" ") + (words.length > 0 ? " " : "")); - } + const handleFilterSelect = (filter: KnowledgeFilterData | null) => { + onFilterSelect(filter); - setIsFilterDropdownOpen(false); - setFilterSearchTerm(""); - setSelectedFilterIndex(0); - }; + // Remove the @searchTerm from the input + const words = input.split(" "); + const lastWord = words[words.length - 1]; - const handleChange = (e: React.ChangeEvent) => { - const newValue = e.target.value; - onChange(newValue); // Call parent's onChange with the string value + if (lastWord.startsWith("@")) { + // Remove the @search term + words.pop(); + onChange(words.join(" ") + (words.length > 0 ? " " : "")); + } - // Find if there's an @ at the start of the last word - const words = newValue.split(" "); - const lastWord = words[words.length - 1]; + setIsFilterDropdownOpen(false); + setFilterSearchTerm(""); + setSelectedFilterIndex(0); + }; - if (lastWord.startsWith("@")) { - const searchTerm = lastWord.slice(1); // Remove the @ - setFilterSearchTerm(searchTerm); - setSelectedFilterIndex(0); + const handleChange = (e: React.ChangeEvent) => { + const newValue = e.target.value; + onChange(newValue); // Call parent's onChange with the string value - // Only set anchor position when @ is first detected (search term is empty) - if (searchTerm === "") { - const getCursorPosition = (textarea: HTMLTextAreaElement) => { - // Create a hidden div with the same styles as the textarea - const div = document.createElement("div"); - const computedStyle = getComputedStyle(textarea); + // Find if there's an @ at the start of the last word + const words = newValue.split(" "); + const lastWord = words[words.length - 1]; - // Copy all computed styles to the hidden div - for (const style of computedStyle) { - (div.style as unknown as Record)[style] = - computedStyle.getPropertyValue(style); - } + if (lastWord.startsWith("@")) { + const searchTerm = lastWord.slice(1); // Remove the @ + setFilterSearchTerm(searchTerm); + setSelectedFilterIndex(0); - // Set the div to be hidden but not un-rendered - div.style.position = "absolute"; - div.style.visibility = "hidden"; - div.style.whiteSpace = "pre-wrap"; - div.style.wordWrap = "break-word"; - div.style.overflow = "hidden"; - div.style.height = "auto"; - div.style.width = `${textarea.getBoundingClientRect().width}px`; + // Only set anchor position when @ is first detected (search term is empty) + if (searchTerm === "") { + const getCursorPosition = (textarea: HTMLTextAreaElement) => { + // Create a hidden div with the same styles as the textarea + const div = document.createElement("div"); + const computedStyle = getComputedStyle(textarea); - // Get the text up to the cursor position - const cursorPos = textarea.selectionStart || 0; - const textBeforeCursor = textarea.value.substring(0, cursorPos); + // Copy all computed styles to the hidden div + for (const style of computedStyle) { + (div.style as unknown as Record)[style] = + computedStyle.getPropertyValue(style); + } - // Add the text before cursor - div.textContent = textBeforeCursor; + // Set the div to be hidden but not un-rendered + div.style.position = "absolute"; + div.style.visibility = "hidden"; + div.style.whiteSpace = "pre-wrap"; + div.style.wordWrap = "break-word"; + div.style.overflow = "hidden"; + div.style.height = "auto"; + div.style.width = `${textarea.getBoundingClientRect().width}px`; - // Create a span to mark the end position - const span = document.createElement("span"); - span.textContent = "|"; // Cursor marker - div.appendChild(span); + // Get the text up to the cursor position + const cursorPos = textarea.selectionStart || 0; + const textBeforeCursor = textarea.value.substring(0, cursorPos); - // Add the text after cursor to handle word wrapping - const textAfterCursor = textarea.value.substring(cursorPos); - div.appendChild(document.createTextNode(textAfterCursor)); + // Add the text before cursor + div.textContent = textBeforeCursor; - // Add the div to the document temporarily - document.body.appendChild(div); + // Create a span to mark the end position + const span = document.createElement("span"); + span.textContent = "|"; // Cursor marker + div.appendChild(span); - // Get positions - const inputRect = textarea.getBoundingClientRect(); - const divRect = div.getBoundingClientRect(); - const spanRect = span.getBoundingClientRect(); + // Add the text after cursor to handle word wrapping + const textAfterCursor = textarea.value.substring(cursorPos); + div.appendChild(document.createTextNode(textAfterCursor)); - // Calculate the cursor position relative to the input - const x = inputRect.left + (spanRect.left - divRect.left); - const y = inputRect.top + (spanRect.top - divRect.top); + // Add the div to the document temporarily + document.body.appendChild(div); - // Clean up - document.body.removeChild(div); + // Get positions + const inputRect = textarea.getBoundingClientRect(); + const divRect = div.getBoundingClientRect(); + const spanRect = span.getBoundingClientRect(); - return { x, y }; - }; + // Calculate the cursor position relative to the input + const x = inputRect.left + (spanRect.left - divRect.left); + const y = inputRect.top + (spanRect.top - divRect.top); - const pos = getCursorPosition(e.target); - setAnchorPosition(pos); - } + // Clean up + document.body.removeChild(div); - if (!isFilterDropdownOpen) { - setIsFilterDropdownOpen(true); - } - } else if (isFilterDropdownOpen) { - // Close dropdown if @ is no longer present - setIsFilterDropdownOpen(false); - setFilterSearchTerm(""); - } - }; + return { x, y }; + }; - const handleKeyDown = (e: React.KeyboardEvent) => { - if (isFilterDropdownOpen) { - if (e.key === "Escape") { - e.preventDefault(); - setIsFilterDropdownOpen(false); - setFilterSearchTerm(""); - setSelectedFilterIndex(0); - inputRef.current?.focus(); - return; - } + const pos = getCursorPosition(e.target); + setAnchorPosition(pos); + } - if (e.key === "ArrowDown") { - e.preventDefault(); - setSelectedFilterIndex((prev) => - prev < filteredFilters.length - 1 ? prev + 1 : 0, - ); - return; - } + if (!isFilterDropdownOpen) { + setIsFilterDropdownOpen(true); + } + } else if (isFilterDropdownOpen) { + // Close dropdown if @ is no longer present + setIsFilterDropdownOpen(false); + setFilterSearchTerm(""); + } + }; - if (e.key === "ArrowUp") { - e.preventDefault(); - setSelectedFilterIndex((prev) => - prev > 0 ? prev - 1 : filteredFilters.length - 1, - ); - return; - } + const handleKeyDown = (e: React.KeyboardEvent) => { + if (isFilterDropdownOpen) { + if (e.key === "Escape") { + e.preventDefault(); + setIsFilterDropdownOpen(false); + setFilterSearchTerm(""); + setSelectedFilterIndex(0); + inputRef.current?.focus(); + return; + } - if (e.key === "Enter") { - // Check if we're at the end of an @ mention - const cursorPos = e.currentTarget.selectionStart || 0; - const textBeforeCursor = input.slice(0, cursorPos); - const words = textBeforeCursor.split(" "); - const lastWord = words[words.length - 1]; + if (e.key === "ArrowDown") { + e.preventDefault(); + setSelectedFilterIndex((prev) => + prev < filteredFilters.length - 1 ? prev + 1 : 0, + ); + return; + } - if ( - lastWord.startsWith("@") && - filteredFilters[selectedFilterIndex] - ) { - e.preventDefault(); - handleFilterSelect(filteredFilters[selectedFilterIndex]); - return; - } - } + if (e.key === "ArrowUp") { + e.preventDefault(); + setSelectedFilterIndex((prev) => + prev > 0 ? prev - 1 : filteredFilters.length - 1, + ); + return; + } - if (e.key === " ") { - // Select filter on space if we're typing an @ mention - const cursorPos = e.currentTarget.selectionStart || 0; - const textBeforeCursor = input.slice(0, cursorPos); - const words = textBeforeCursor.split(" "); - const lastWord = words[words.length - 1]; + if (e.key === "Enter") { + // Check if we're at the end of an @ mention + const cursorPos = e.currentTarget.selectionStart || 0; + const textBeforeCursor = input.slice(0, cursorPos); + const words = textBeforeCursor.split(" "); + const lastWord = words[words.length - 1]; - if ( - lastWord.startsWith("@") && - filteredFilters[selectedFilterIndex] - ) { - e.preventDefault(); - handleFilterSelect(filteredFilters[selectedFilterIndex]); - return; - } - } - } + if ( + lastWord.startsWith("@") && + filteredFilters[selectedFilterIndex] + ) { + e.preventDefault(); + handleFilterSelect(filteredFilters[selectedFilterIndex]); + return; + } + } - // Pass through to parent onKeyDown for other key handling - onKeyDown(e); - }; + if (e.key === " ") { + // Select filter on space if we're typing an @ mention + const cursorPos = e.currentTarget.selectionStart || 0; + const textBeforeCursor = input.slice(0, cursorPos); + const words = textBeforeCursor.split(" "); + const lastWord = words[words.length - 1]; - return ( -
-
- {/* Outer container - flex-col to stack file preview above input */} -
- - {/* File Preview Section - Always above */} - - {uploadedFile && ( - - { - onFileSelected(null); - }} - isUploading={isUploading} - /> - - )} - - - {isDragging && ( - -

- Add files to conversation -

-

- Text formats and image files.{" "} - 10 files per chat,{" "} - 150 MB each. -

- - )} - - {/* Main Input Container - flex-row or flex-col based on textarea height */} -
40 ? "flex-col" : "flex-row items-center" - }`} - > - {/* Filter + Textarea Section */} -
40 ? "w-full" : "flex-1"}`} - > - {textareaHeight <= 40 && - (selectedFilter ? ( - { - setSelectedFilter(null); - setIsFilterHighlighted(false); - }} - /> - ) : ( - - ))} -
- setTextareaHeight(height)} - maxRows={7} - autoComplete="off" - minRows={1} - placeholder="Ask a question..." - disabled={loading} - className={`w-full text-sm bg-transparent focus-visible:outline-none resize-none`} - rows={1} - /> -
-
+ if ( + lastWord.startsWith("@") && + filteredFilters[selectedFilterIndex] + ) { + e.preventDefault(); + handleFilterSelect(filteredFilters[selectedFilterIndex]); + return; + } + } + } - {/* Action Buttons Section */} -
40 ? "justify-between w-full" : ""}`} - > - {textareaHeight > 40 && - (selectedFilter ? ( - { - setSelectedFilter(null); - setIsFilterHighlighted(false); - }} - /> - ) : ( - - ))} -
- - -
-
-
-
- + // Pass through to parent onKeyDown for other key handling + onKeyDown(e); + }; - { - setIsFilterDropdownOpen(open); - }} - > - {anchorPosition && ( - -
- - )} - { - // Prevent auto focus on the popover content - e.preventDefault(); - // Keep focus on the input - }} - > -
- {filterSearchTerm && ( -
- Searching: @{filterSearchTerm} -
- )} - {availableFilters.length === 0 ? ( -
- No knowledge filters available -
- ) : ( - <> - {!filterSearchTerm && ( - - )} - {filteredFilters.map((filter, index) => ( - - ))} - {filteredFilters.length === 0 && filterSearchTerm && ( -
- No filters match "{filterSearchTerm}" -
- )} - - )} -
- - - -
- ); - }, + return ( +
+
+ {/* Outer container - flex-col to stack file preview above input */} +
+ + {/* File Preview Section - Always above */} + + {uploadedFile && ( + + { + onFileSelected(null); + }} + isUploading={isUploading} + /> + + )} + + + {isDragging && ( + +

+ Add files to conversation +

+

+ Text formats and image files.{" "} + 10 files per chat,{" "} + 150 MB each. +

+ + )} + + {/* Main Input Container - flex-row or flex-col based on textarea height */} +
40 ? "flex-col" : "flex-row items-center" + }`} + > + {/* Filter + Textarea Section */} +
40 ? "w-full" : "flex-1"}`} + > + {textareaHeight <= 40 && + (selectedFilter ? ( + { + setSelectedFilter(null); + setIsFilterHighlighted(false); + }} + /> + ) : ( + + ))} +
+ setTextareaHeight(height)} + maxRows={7} + autoComplete="off" + minRows={1} + placeholder="Ask a question..." + disabled={loading} + className={`w-full text-sm bg-transparent focus-visible:outline-none resize-none`} + rows={1} + /> +
+
+ + {/* Action Buttons Section */} +
40 ? "justify-between w-full" : ""}`} + > + {textareaHeight > 40 && + (selectedFilter ? ( + { + setSelectedFilter(null); + setIsFilterHighlighted(false); + }} + /> + ) : ( + + ))} +
+ + +
+
+
+
+ + + { + setIsFilterDropdownOpen(open); + }} + > + {anchorPosition && ( + +
+ + )} + { + // Prevent auto focus on the popover content + e.preventDefault(); + // Keep focus on the input + }} + > +
+ {filterSearchTerm && ( +
+ Searching: @{filterSearchTerm} +
+ )} + {allFilters.length === 0 ? ( +
+ No knowledge filters available +
+ ) : ( + <> + {!filterSearchTerm && ( + + )} + {filteredFilters.map((filter, index) => ( + + ))} + {filteredFilters.length === 0 && filterSearchTerm && ( +
+ No filters match "{filterSearchTerm}" +
+ )} + + )} +
+ + + +
+ ); + }, ); ChatInput.displayName = "ChatInput"; diff --git a/frontend/components/discord-link.tsx b/frontend/components/discord-link.tsx deleted file mode 100644 index 584daa26..00000000 --- a/frontend/components/discord-link.tsx +++ /dev/null @@ -1,47 +0,0 @@ -"use client"; - -import * as React from "react"; -import { cn } from "@/lib/utils"; -import { useDiscordMembers } from "@/hooks/use-discord-members"; -import { formatCount } from "@/lib/format-count"; - -interface DiscordLinkProps { - inviteCode?: string; - className?: string; -} - -const DiscordLink = React.forwardRef( - ({ inviteCode = "EqksyE2EX9", className }, ref) => { - const { data, isLoading, error } = useDiscordMembers(inviteCode); - - return ( - - - - - - {isLoading - ? "..." - : error - ? "--" - : data - ? formatCount(data.approximate_member_count) - : "--"} - - - ); - }, -); - -DiscordLink.displayName = "DiscordLink"; - -export { DiscordLink }; diff --git a/frontend/components/file-upload-area.tsx b/frontend/components/file-upload-area.tsx deleted file mode 100644 index 84f0806a..00000000 --- a/frontend/components/file-upload-area.tsx +++ /dev/null @@ -1,103 +0,0 @@ -"use client"; - -import * as React from "react"; -import { cn } from "@/lib/utils"; -import { Button } from "@/components/ui/button"; -import { Loader2 } from "lucide-react"; - -interface FileUploadAreaProps { - onFileSelected?: (file: File) => void; - isLoading?: boolean; - className?: string; -} - -const FileUploadArea = React.forwardRef( - ({ onFileSelected, isLoading = false, className }, ref) => { - const [isDragging, setIsDragging] = React.useState(false); - const fileInputRef = React.useRef(null); - - const handleDragOver = (e: React.DragEvent) => { - e.preventDefault(); - setIsDragging(true); - }; - - const handleDragLeave = (e: React.DragEvent) => { - e.preventDefault(); - setIsDragging(false); - }; - - const handleDrop = (e: React.DragEvent) => { - e.preventDefault(); - setIsDragging(false); - - const files = Array.from(e.dataTransfer.files); - if (files.length > 0 && onFileSelected) { - onFileSelected(files[0]); - } - }; - - const handleFileSelect = (e: React.ChangeEvent) => { - const files = Array.from(e.target.files || []); - if (files.length > 0 && onFileSelected) { - onFileSelected(files[0]); - } - }; - - const handleClick = () => { - if (!isLoading) { - fileInputRef.current?.click(); - } - }; - - return ( -
- - -
- {isLoading && ( -
- -
- )} - -
-

- {isLoading - ? "Processing file..." - : "Drop files here or click to upload"} -

-

- {isLoading - ? "Please wait while your file is being processed" - : ""} -

-
- - {!isLoading && } -
-
- ); - }, -); - -FileUploadArea.displayName = "FileUploadArea"; - -export { FileUploadArea }; diff --git a/frontend/components/github-star-button.tsx b/frontend/components/github-star-button.tsx deleted file mode 100644 index 81e4ca98..00000000 --- a/frontend/components/github-star-button.tsx +++ /dev/null @@ -1,47 +0,0 @@ -"use client"; - -import * as React from "react"; -import { cn } from "@/lib/utils"; -import { Github } from "lucide-react"; -import { useGitHubStars } from "@/hooks/use-github-stars"; -import { formatCount } from "@/lib/format-count"; - -interface GitHubStarButtonProps { - repo?: string; - className?: string; -} - -const GitHubStarButton = React.forwardRef< - HTMLAnchorElement, - GitHubStarButtonProps ->(({ repo = "phact/openrag", className }, ref) => { - const { data, isLoading, error } = useGitHubStars(repo); - - return ( - - - - {isLoading - ? "..." - : error - ? "--" - : data - ? formatCount(data.stargazers_count) - : "--"} - - - ); -}); - -GitHubStarButton.displayName = "GitHubStarButton"; - -export { GitHubStarButton }; diff --git a/frontend/components/knowledge-filter-dropdown.tsx b/frontend/components/knowledge-filter-dropdown.tsx deleted file mode 100644 index cb2106d9..00000000 --- a/frontend/components/knowledge-filter-dropdown.tsx +++ /dev/null @@ -1,458 +0,0 @@ -"use client"; - -import { useState, useEffect, useRef } from "react"; -import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { Card, CardContent } from "@/components/ui/card"; - -import { Label } from "@/components/ui/label"; -import { Textarea } from "@/components/ui/textarea"; -import { - ChevronDown, - Filter, - Search, - X, - Loader2, - Plus, - Save, -} from "lucide-react"; -import { cn } from "@/lib/utils"; - -interface KnowledgeFilter { - id: string; - name: string; - description: string; - query_data: string; - owner: string; - created_at: string; - updated_at: string; -} - -interface ParsedQueryData { - query: string; - filters: { - data_sources: string[]; - document_types: string[]; - owners: string[]; - }; - limit: number; - scoreThreshold: number; -} - -interface KnowledgeFilterDropdownProps { - selectedFilter: KnowledgeFilter | null; - onFilterSelect: (filter: KnowledgeFilter | null) => void; -} - -export function KnowledgeFilterDropdown({ - selectedFilter, - onFilterSelect, -}: KnowledgeFilterDropdownProps) { - const [isOpen, setIsOpen] = useState(false); - const [filters, setFilters] = useState([]); - const [loading, setLoading] = useState(false); - const [searchQuery, setSearchQuery] = useState(""); - const [showCreateModal, setShowCreateModal] = useState(false); - const [createName, setCreateName] = useState(""); - const [createDescription, setCreateDescription] = useState(""); - const [creating, setCreating] = useState(false); - const dropdownRef = useRef(null); - - const loadFilters = async (query = "") => { - setLoading(true); - try { - const response = await fetch("/api/knowledge-filter/search", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - query, - limit: 20, // Limit for dropdown - }), - }); - - const result = await response.json(); - if (response.ok && result.success) { - setFilters(result.filters); - } else { - console.error("Failed to load knowledge filters:", result.error); - setFilters([]); - } - } catch (error) { - console.error("Error loading knowledge filters:", error); - setFilters([]); - } finally { - setLoading(false); - } - }; - - const deleteFilter = async (filterId: string, e: React.MouseEvent) => { - e.stopPropagation(); - - try { - const response = await fetch(`/api/knowledge-filter/${filterId}`, { - method: "DELETE", - }); - - if (response.ok) { - // Remove from local state - setFilters((prev) => prev.filter((f) => f.id !== filterId)); - - // If this was the selected filter, clear selection - if (selectedFilter?.id === filterId) { - onFilterSelect(null); - } - } else { - console.error("Failed to delete knowledge filter"); - } - } catch (error) { - console.error("Error deleting knowledge filter:", error); - } - }; - - const handleFilterSelect = (filter: KnowledgeFilter) => { - onFilterSelect(filter); - setIsOpen(false); - }; - - const handleClearFilter = () => { - onFilterSelect(null); - setIsOpen(false); - }; - - const handleCreateNew = () => { - setIsOpen(false); - setShowCreateModal(true); - }; - - const handleCreateFilter = async () => { - if (!createName.trim()) return; - - setCreating(true); - try { - // Create a basic filter with wildcards (match everything by default) - const defaultFilterData = { - query: "", - filters: { - data_sources: ["*"], - document_types: ["*"], - owners: ["*"], - }, - limit: 10, - scoreThreshold: 0, - }; - - const response = await fetch("/api/knowledge-filter", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - name: createName.trim(), - description: createDescription.trim(), - queryData: JSON.stringify(defaultFilterData), - }), - }); - - const result = await response.json(); - if (response.ok && result.success) { - // Create the new filter object - const newFilter: KnowledgeFilter = { - id: result.filter.id, - name: createName.trim(), - description: createDescription.trim(), - query_data: JSON.stringify(defaultFilterData), - owner: result.filter.owner, - created_at: result.filter.created_at, - updated_at: result.filter.updated_at, - }; - - // Add to local filters list - setFilters((prev) => [newFilter, ...prev]); - - // Select the new filter - onFilterSelect(newFilter); - - // Close modal and reset form - setShowCreateModal(false); - setCreateName(""); - setCreateDescription(""); - } else { - console.error("Failed to create knowledge filter:", result.error); - } - } catch (error) { - console.error("Error creating knowledge filter:", error); - } finally { - setCreating(false); - } - }; - - const handleCancelCreate = () => { - setShowCreateModal(false); - setCreateName(""); - setCreateDescription(""); - }; - - const getFilterSummary = (filter: KnowledgeFilter): string => { - try { - const parsed = JSON.parse(filter.query_data) as ParsedQueryData; - const parts = []; - - if (parsed.query) parts.push(`"${parsed.query}"`); - if (parsed.filters.data_sources.length > 0) - parts.push(`${parsed.filters.data_sources.length} sources`); - if (parsed.filters.document_types.length > 0) - parts.push(`${parsed.filters.document_types.length} types`); - if (parsed.filters.owners.length > 0) - parts.push(`${parsed.filters.owners.length} owners`); - - return parts.join(" • ") || "No filters"; - } catch { - return "Invalid filter"; - } - }; - - useEffect(() => { - if (isOpen) { - loadFilters(); - } - }, [isOpen]); - - useEffect(() => { - const timeoutId = setTimeout(() => { - if (isOpen) { - loadFilters(searchQuery); - } - }, 300); - - return () => clearTimeout(timeoutId); - }, [searchQuery, isOpen]); - - // Close dropdown when clicking outside - useEffect(() => { - const handleClickOutside = (event: MouseEvent) => { - if ( - dropdownRef.current && - !dropdownRef.current.contains(event.target as Node) - ) { - setIsOpen(false); - } - }; - - document.addEventListener("mousedown", handleClickOutside); - return () => document.removeEventListener("mousedown", handleClickOutside); - }, []); - - return ( -
- - - {isOpen && ( - - - {/* Search Header */} -
-
- - setSearchQuery(e.target.value)} - className="pl-9 h-8 text-sm" - /> -
-
- - {/* Filter List */} -
- {/* Clear filter option */} -
-
- -
-
All Knowledge
-
- No filters applied -
-
-
-
- - {loading ? ( -
- - - Loading... - -
- ) : filters.length === 0 ? ( -
- {searchQuery ? "No filters found" : "No saved filters"} -
- ) : ( - filters.map((filter) => ( -
handleFilterSelect(filter)} - className={cn( - "flex items-center gap-3 p-3 hover:bg-accent hover:text-accent-foreground cursor-pointer group transition-colors", - selectedFilter?.id === filter.id && - "bg-accent text-accent-foreground", - )} - > -
- -
-
- {filter.name} -
-
- {getFilterSummary(filter)} -
-
-
- -
- )) - )} -
- - {/* Create New Filter Option */} -
-
- -
-
- Create New Filter -
-
- Save current search as filter -
-
-
-
- - {/* Selected Filter Details */} - {selectedFilter && ( -
-
- Selected: {selectedFilter.name} -
- {selectedFilter.description && ( -
- {selectedFilter.description} -
- )} -
- )} - - - )} - - {/* Create Filter Modal */} - {showCreateModal && ( -
-
-

- Create New Knowledge Filter -

- -
-
- - setCreateName(e.target.value)} - className="mt-1" - /> -
- -
- -