gmakstutis/ragflow
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Feat: Enhance MinerU options with language selection and parsing configurations

Browse source
This commit is contained in:
user210 2025-12-12 20:07:34 +02:00
parent 948bc93786
commit 03d22b9e71
13 changed files with 246 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
api/db/services/task_service.py
View file

@ -121,6 +121,13 @@ class TaskService(CommonService):
.where(cls.model.id == task_id)
)
docs = list(docs.dicts())
# Assuming docs = list(docs.dicts())
if docs:
kb_config = docs[0]['kb_parser_config'] # Dict from Knowledgebase.parser_config
mineru_method = kb_config.get('mineru_parse_method', 'auto')
mineru_formula = kb_config.get('mineru_formula_enable', True)
mineru_table = kb_config.get('mineru_table_enable', True)
print(mineru_method, mineru_formula, mineru_table)
if not docs:
return None

34
deepdoc/parser/mineru_parser.py
View file

@ -53,6 +53,30 @@ class MinerUContentType(StrEnum):
DISCARDED = "discarded"
# Mapping from language names to MinerU language codes
LANGUAGE_TO_MINERU_MAP = {
'English': 'en',
'Chinese': 'ch',
'Traditional Chinese': 'chinese_cht',
'Russian': 'east_slavic',
'Ukrainian': 'east_slavic',
'Indonesian': 'latin',
'Spanish': 'latin',
'Vietnamese': 'latin',
'Japanese': 'japan',
'Korean': 'korean',
'Portuguese BR': 'latin',
'German': 'latin',
'French': 'latin',
'Italian': 'latin',
'Tamil': 'ta',
'Telugu': 'te',
'Kannada': 'ka',
'Thai': 'th',
'Greek': 'el',
'Hindi': 'devanagari',
}
class MinerUParser(RAGFlowPdfParser):
def __init__(self, mineru_path: str = "mineru", mineru_api: str = "", mineru_server_url: str = ""):
self.mineru_path = Path(mineru_path)
@ -576,17 +600,21 @@ class MinerUParser(RAGFlowPdfParser):
*,
output_dir: Optional[str] = None,
backend: str = "pipeline",
lang: Optional[str] = None,
method: str = "auto",
server_url: Optional[str] = None,
delete_output: bool = True,
parse_method: str = "raw",
**kwargs,
) -> tuple:
import shutil
temp_pdf = None
created_tmp_dir = False
# Assuming the dict is defined as shown
lang = kwargs.get('lang', 'English')
mineru_lang_code = LANGUAGE_TO_MINERU_MAP.get(lang, 'ch') # Returns 'ch' if lang not found
# Use .get() to avoid KeyError if the key doesn't exist
# remove spaces, or mineru crash, and _read_output fail too
file_path = Path(filepath)
pdf_file_name = file_path.stem.replace(" ", "") + ".pdf"
@ -625,7 +653,7 @@ class MinerUParser(RAGFlowPdfParser):
self.__images__(pdf, zoomin=1)
try:
self._run_mineru(pdf, out_dir, method=method, backend=backend, lang=lang, server_url=server_url, callback=callback)
self._run_mineru(pdf, out_dir, method=method, backend=backend, lang=mineru_lang_code, server_url=server_url, callback=callback)
outputs = self._read_output(out_dir, pdf.stem, method=method, backend=backend)
self.logger.info(f"[MinerU] Parsed {len(outputs)} blocks from PDF.")
if callback:

2
rag/app/naive.py
View file

@ -85,6 +85,8 @@ def by_mineru(filename, binary=None, from_page=0, to_page=100000, lang="Chinese"
binary=binary,
callback=callback,
parse_method=parse_method,
lang=lang,
**kwargs
)
return sections, tables, pdf_parser
except Exception as e:

3
rag/llm/ocr_model.py
View file

@ -57,7 +57,7 @@ class MinerUOcrModel(Base, MinerUParser):
server_url = server_url or self.mineru_server_url
return self.check_installation(backend=backend, server_url=server_url)
def parse_pdf(self, filepath: str, binary=None, callback=None, parse_method: str = "raw", **kwargs):
def parse_pdf(self, filepath: str, binary=None, callback=None, parse_method: str = "raw",**kwargs):
ok, reason = self.check_available()
if not ok:
raise RuntimeError(f"MinerU not found or server not accessible: {reason}. Please install it via: pip install -U 'mineru[core]'.")
@ -72,5 +72,6 @@ class MinerUOcrModel(Base, MinerUParser):
server_url=self.mineru_server_url,
delete_output=self.mineru_delete_output,
parse_method=parse_method,
**kwargs
)
return sections, tables

97
web/src/components/mineru-options-form-field.tsx Normal file
View file

@ -0,0 +1,97 @@
import { RAGFlowFormItem } from '@/components/ragflow-form';
import { RAGFlowSelect } from '@/components/ui/select';
import { Switch } from '@/components/ui/switch';
import { LLMFactory } from '@/constants/llm';
import { buildOptions } from '@/utils/form';
import { useFormContext, useWatch } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
const parseMethodOptions = buildOptions(['auto', 'txt', 'ocr']);
export function MinerUOptionsFormField() {
const form = useFormContext();
const { t } = useTranslation();
const layoutRecognize = useWatch({
control: form.control,
name: 'parser_config.layout_recognize',
});
// Check if MinerU is selected (the value contains 'MinerU' or matches the factory name)
const isMinerUSelected =
layoutRecognize?.includes(LLMFactory.MinerU) ||
layoutRecognize?.toLowerCase()?.includes('mineru');
if (!isMinerUSelected) {
return null;
}
return (
<div className="space-y-4 border-l-2 border-primary/30 pl-4 ml-2">
<div className="text-sm font-medium text-text-secondary">
{t('knowledgeConfiguration.mineruOptions', 'MinerU Options')}
</div>
<RAGFlowFormItem
name="parser_config.mineru_parse_method"
label={t('knowledgeConfiguration.mineruParseMethod', 'Parse Method')}
tooltip={t(
'knowledgeConfiguration.mineruParseMethodTip',
'Method for parsing PDF: auto (automatic detection), txt (text extraction), ocr (optical character recognition)',
)}
horizontal={true}
>
{(field) => (
<RAGFlowSelect
value={field.value || 'auto'}
onChange={field.onChange}
options={parseMethodOptions}
placeholder={t('common.selectPlaceholder', 'Select value')}
/>
)}
</RAGFlowFormItem>
<RAGFlowFormItem
name="parser_config.mineru_formula_enable"
label={t(
'knowledgeConfiguration.mineruFormulaEnable',
'Formula Recognition',
)}
tooltip={t(
'knowledgeConfiguration.mineruFormulaEnableTip',
'Enable formula recognition. Note: This may not work correctly for Cyrillic documents.',
)}
horizontal={true}
labelClassName="!mb-0"
>
{(field) => (
<Switch
checked={field.value ?? true}
onCheckedChange={field.onChange}
/>
)}
</RAGFlowFormItem>
<RAGFlowFormItem
name="parser_config.mineru_table_enable"
label={t(
'knowledgeConfiguration.mineruTableEnable',
'Table Recognition',
)}
tooltip={t(
'knowledgeConfiguration.mineruTableEnableTip',
'Enable table recognition and extraction.',
)}
horizontal={true}
labelClassName="!mb-0"
>
{(field) => (
<Switch
checked={field.value ?? true}
onCheckedChange={field.onChange}
/>
)}
</RAGFlowFormItem>
</div>
);
}

10
web/src/constants/common.ts
View file

@ -103,14 +103,22 @@ export const LanguageTranslationMap = {
Chinese: 'zh',
'Traditional Chinese': 'zh-TRADITIONAL',
Russian: 'ru',
Indonesia: 'id',
Indonesian: 'id',
Spanish: 'es',
Vietnamese: 'vi',
Japanese: 'ja',
Korean: 'ko',
'Portuguese BR': 'pt-br',
German: 'de',
French: 'fr',
Italian: 'it',
Tamil: 'ta',
Telugu: 'te',
Kannada: 'ka',
Thai: 'th',
Greek: 'el',
Hindi: 'hi',
Ukrainian: 'uk',
};
export enum FileMimeType {

9
web/src/locales/en.ts
View file

@ -329,6 +329,15 @@ export default {
reRankModelWaring: 'Re-rank model is very time consuming.',
},
knowledgeConfiguration: {
mineruOptions: 'MinerU Options',
mineruParseMethod: 'Parse Method',
mineruParseMethodTip:
'Method for parsing PDF: auto (automatic detection), txt (text extraction), ocr (optical character recognition)',
mineruFormulaEnable: 'Formula Recognition',
mineruFormulaEnableTip:
'Enable formula recognition. Note: This may not work correctly for Cyrillic documents.',
mineruTableEnable: 'Table Recognition',
mineruTableEnableTip: 'Enable table recognition and extraction.',
overlappedPercent: 'Overlapped percent',
generationScopeTip:
'Determines whether RAPTOR is generated for the entire dataset or for a single file.',

9
web/src/locales/zh.ts
View file

@ -264,6 +264,15 @@ export default {
theDocumentBeingParsedCannotBeDeleted: '正在解析的文档不能被删除',
},
knowledgeConfiguration: {
mineruOptions: 'MinerU 选项',
mineruParseMethod: '解析方法',
mineruParseMethodTip:
'PDF 解析方法：auto（自动检测）、txt（文本提取）、ocr（光学字符识别）',
mineruFormulaEnable: '公式识别',
mineruFormulaEnableTip:
'启用公式识别。注意：对于西里尔文档可能无法正常工作。',
mineruTableEnable: '表格识别',
mineruTableEnableTip: '启用表格识别和提取。',
generationScopeTip: '选择 RAPTOR 的生成范围：整个知识库或单个文件。',
generationScope: '生成范围',
scopeSingleFile: '单文件',

2
web/src/pages/dataset/dataset-setting/configuration/naive.tsx
View file

@ -6,6 +6,7 @@ import { DelimiterFormField } from '@/components/delimiter-form-field';
import { ExcelToHtmlFormField } from '@/components/excel-to-html-form-field';
import { LayoutRecognizeFormField } from '@/components/layout-recognize-form-field';
import { MaxTokenNumberFormField } from '@/components/max-token-number-from-field';
import { MinerUOptionsFormField } from '@/components/mineru-options-form-field';
import {
ConfigurationFormContainer,
MainContainer,
@ -17,6 +18,7 @@ export function NaiveConfiguration() {
<MainContainer>
<ConfigurationFormContainer>
<LayoutRecognizeFormField></LayoutRecognizeFormField>
<MinerUOptionsFormField></MinerUOptionsFormField>
<MaxTokenNumberFormField initialValue={512}></MaxTokenNumberFormField>
<DelimiterFormField></DelimiterFormField>
<EnableTocToggle />

5
web/src/pages/dataset/dataset-setting/form-schema.ts
View file

@ -13,6 +13,7 @@ export const formSchema = z
// avatar: z.instanceof(File),
avatar: z.any().nullish(),
permission: z.string().optional(),
language: z.string().optional(),
parser_id: z.string(),
pipeline_id: z.string().optional(),
pipeline_name: z.string().optional(),
@ -30,6 +31,10 @@ export const formSchema = z
topn_tags: z.number().optional(),
toc_extraction: z.boolean().optional(),
overlapped_percent: z.number().optional(),
// MinerU-specific options
mineru_parse_method: z.enum(['auto', 'txt', 'ocr']).optional(),
mineru_formula_enable: z.boolean().optional(),
mineru_table_enable: z.boolean().optional(),
raptor: z
.object({
use_raptor: z.boolean().optional(),

23
web/src/pages/dataset/dataset-setting/general-form.tsx
View file

@ -1,5 +1,7 @@
import { AvatarUpload } from '@/components/avatar-upload';
import { SelectWithSearch } from '@/components/originui/select-with-search';
import PageRankFormField from '@/components/page-rank-form-field';
import { RAGFlowFormItem } from '@/components/ragflow-form';
import {
FormControl,
FormField,
@ -8,6 +10,8 @@ import {
FormMessage,
} from '@/components/ui/form';
import { Input } from '@/components/ui/input';
import { LanguageTranslationMap } from '@/constants/common';
import { useMemo } from 'react';
import { useFormContext } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { TagItems } from './components/tag-item';
@ -18,6 +22,13 @@ export function GeneralForm() {
const form = useFormContext();
const { t } = useTranslation();
const languageOptions = useMemo(() => {
return Object.keys(LanguageTranslationMap).map((x) => ({
label: x,
value: x,
}));
}, []);
return (
<>
<FormField
@ -41,6 +52,18 @@ export function GeneralForm() {
</FormItem>
)}
/>
<div className="items-center">
<RAGFlowFormItem
name="language"
label={t('common.language')}
horizontal={true}
>
<SelectWithSearch
options={languageOptions}
triggerClassName="w-full"
></SelectWithSearch>
</RAGFlowFormItem>
</div>
<FormField
control={form.control}
name="avatar"

5
web/src/pages/dataset/dataset-setting/index.tsx
View file

@ -58,6 +58,7 @@ export default function DatasetSettings() {
name: '',
parser_id: DocumentParserType.Naive,
permission: PermissionRole.Me,
language: 'English',
parser_config: {
layout_recognize: DocumentType.DeepDOC,
chunk_token_num: 512,
@ -68,6 +69,10 @@ export default function DatasetSettings() {
topn_tags: 3,
toc_extraction: false,
overlapped_percent: 0,
// MinerU-specific defaults
mineru_parse_method: 'auto',
mineru_formula_enable: true,
mineru_table_enable: true,
raptor: {
use_raptor: true,
max_token: 256,

46
web/src/pages/datasets/dataset-creating-dialog.tsx
View file

@ -16,10 +16,18 @@ import {
FormMessage,
} from '@/components/ui/form';
import { Input } from '@/components/ui/input';
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from '@/components/ui/select';
import { LanguageTranslationMap } from '@/constants/common';
import { FormLayout } from '@/constants/form';
import { IModalProps } from '@/interfaces/common';
import { zodResolver } from '@hookform/resolvers/zod';
import { useEffect } from 'react';
import { useEffect, useMemo } from 'react';
import { useForm, useWatch } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
@ -34,6 +42,13 @@ const FormId = 'dataset-creating-form';
export function InputForm({ onOk }: IModalProps<any>) {
const { t } = useTranslation();
const languageOptions = useMemo(() => {
return Object.keys(LanguageTranslationMap).map((x) => ({
label: x,
value: x,
}));
}, []);
const FormSchema = z
.object({
name: z
@ -51,6 +66,7 @@ export function InputForm({ onOk }: IModalProps<any>) {
.trim(),
parser_id: z.string().optional(),
pipeline_id: z.string().optional(),
language: z.string().optional(),
})
.superRefine((data, ctx) => {
// When parseType === 1, parser_id is required
@ -83,6 +99,7 @@ export function InputForm({ onOk }: IModalProps<any>) {
parseType: 1,
parser_id: '',
embd_id: '',
language: 'English',
},
});
@ -130,6 +147,33 @@ export function InputForm({ onOk }: IModalProps<any>) {
)}
/>
<FormField
control={form.control}
name="language"
render={({ field }) => (
<FormItem>
<FormLabel>{t('common.language')}</FormLabel>
<Select onValueChange={field.onChange} defaultValue={field.value}>
<FormControl>
<SelectTrigger>
<SelectValue
placeholder={t('common.languagePlaceholder')}
/>
</SelectTrigger>
</FormControl>
<SelectContent>
{languageOptions.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
<EmbeddingModelItem line={2} isEdit={false} />
<ParseTypeItem />
{parseType === 1 && <ChunkMethodItem></ChunkMethodItem>}