mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-04-30 15:27:48 +08:00
Feat: support epub parsing (#13650)
Closes #1398 ### What problem does this PR solve? Adds native support for EPUB files. EPUB content is extracted in spine (reading) order and parsed using the existing HTML parser. No new dependencies required. ### Type of change - [x] New Feature (non-breaking change which adds functionality) To check this parser manually: ```python uv run --python 3.12 python -c " from deepdoc.parser import EpubParser with open('$HOME/some_epub_book.epub', 'rb') as f: data = f.read() sections = EpubParser()(None, binary=data, chunk_token_num=512) print(f'Got {len(sections)} sections') for i, s in enumerate(sections[:5]): print(f'\n--- Section {i} ---') print(s[:200]) " ```
This commit is contained in:
@ -43,10 +43,9 @@ from rag.nlp import BULLET_PATTERN, bullets_category, docx_question_level, not_b
|
||||
from rag.utils.base64_image import image2id
|
||||
|
||||
|
||||
|
||||
|
||||
from common.misc_utils import thread_pool_exec
|
||||
|
||||
|
||||
class ParserParam(ProcessParamBase):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@ -82,6 +81,10 @@ class ParserParam(ProcessParamBase):
|
||||
"json",
|
||||
],
|
||||
"video": [],
|
||||
"epub": [
|
||||
"text",
|
||||
"json",
|
||||
],
|
||||
}
|
||||
|
||||
self.setups = {
|
||||
@ -166,6 +169,12 @@ class ParserParam(ProcessParamBase):
|
||||
"output_format": "text",
|
||||
"prompt": "",
|
||||
},
|
||||
"epub": {
|
||||
"suffix": [
|
||||
"epub",
|
||||
],
|
||||
"output_format": "json",
|
||||
},
|
||||
}
|
||||
|
||||
def check(self):
|
||||
@ -219,6 +228,11 @@ class ParserParam(ProcessParamBase):
|
||||
email_output_format = email_config.get("output_format", "")
|
||||
self.check_valid_value(email_output_format, "Email output format abnormal.", self.allowed_output_format["email"])
|
||||
|
||||
epub_config = self.setups.get("epub", "")
|
||||
if epub_config:
|
||||
epub_output_format = epub_config.get("output_format", "")
|
||||
self.check_valid_value(epub_output_format, "EPUB output format abnormal.", self.allowed_output_format["epub"])
|
||||
|
||||
def get_input_form(self) -> dict[str, dict]:
|
||||
return {}
|
||||
|
||||
@ -390,9 +404,7 @@ class Parser(ProcessBase):
|
||||
box = {
|
||||
"text": text,
|
||||
"image": pdf_parser.crop(poss, 1) if isinstance(poss, str) and poss else None,
|
||||
"positions": [[pos[0][-1], *pos[1:]] for pos in pdf_parser.extract_positions(poss)]
|
||||
if isinstance(poss, str) and poss
|
||||
else [],
|
||||
"positions": [[pos[0][-1], *pos[1:]] for pos in pdf_parser.extract_positions(poss)] if isinstance(poss, str) and poss else [],
|
||||
}
|
||||
bboxes.append(box)
|
||||
elif parse_method.lower() == "tcadp parser":
|
||||
@ -698,7 +710,6 @@ class Parser(ProcessBase):
|
||||
markdown_text = docx_parser.to_markdown(name, binary=blob)
|
||||
self.set_output("markdown", markdown_text)
|
||||
|
||||
|
||||
def _slides(self, name, blob, **kwargs):
|
||||
self.callback(random.randint(1, 5) / 100.0, "Start to work on a PowerPoint Document")
|
||||
|
||||
@ -839,11 +850,13 @@ class Parser(ProcessBase):
|
||||
else:
|
||||
txt = cv_model.describe(img_binary.read())
|
||||
|
||||
json_result = [{
|
||||
"text": txt,
|
||||
"image": img,
|
||||
"doc_type_kwd": "image",
|
||||
}]
|
||||
json_result = [
|
||||
{
|
||||
"text": txt,
|
||||
"image": img,
|
||||
"doc_type_kwd": "image",
|
||||
}
|
||||
]
|
||||
self.set_output("json", json_result)
|
||||
|
||||
def _audio(self, name, blob, **kwargs):
|
||||
@ -1013,6 +1026,22 @@ class Parser(ProcessBase):
|
||||
content_txt += fb
|
||||
self.set_output("text", content_txt)
|
||||
|
||||
def _epub(self, name, blob, **kwargs):
|
||||
from deepdoc.parser import EpubParser
|
||||
|
||||
self.callback(random.randint(1, 5) / 100.0, "Start to work on an EPUB.")
|
||||
conf = self._param.setups["epub"]
|
||||
self.set_output("output_format", conf["output_format"])
|
||||
|
||||
epub_parser = EpubParser()
|
||||
sections = epub_parser(name, binary=blob)
|
||||
|
||||
if conf.get("output_format") == "json":
|
||||
json_results = [{"text": s} for s in sections if s]
|
||||
self.set_output("json", json_results)
|
||||
else:
|
||||
self.set_output("text", "\n".join(s for s in sections if s))
|
||||
|
||||
async def _invoke(self, **kwargs):
|
||||
function_map = {
|
||||
"pdf": self._pdf,
|
||||
@ -1024,6 +1053,7 @@ class Parser(ProcessBase):
|
||||
"audio": self._audio,
|
||||
"video": self._video,
|
||||
"email": self._email,
|
||||
"epub": self._epub,
|
||||
}
|
||||
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user