diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index e1f6eaf58d..3869c4ee52 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -729,6 +729,8 @@ class SafeWebBaseLoader(WebBaseLoader): trust_env (bool, optional): set to True if using proxy to make web requests, for example using http(s)_proxy environment variables. Defaults to False. """ + # lxml parses scraped pages far faster than the html.parser default + kwargs.setdefault('default_parser', 'lxml') super().__init__(*args, **kwargs) self.trust_env = trust_env diff --git a/backend/requirements.txt b/backend/requirements.txt index a2c11ad5b6..c6cfb50507 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -77,6 +77,7 @@ unstructured==0.22.31 nltk==3.9.4 Markdown==3.10.2 beautifulsoup4==4.14.3 +lxml==6.1.1 pypandoc==1.17 pandas==3.0.3 openpyxl==3.1.5 diff --git a/pyproject.toml b/pyproject.toml index 7fee1b560e..ef3003da2d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ dependencies = [ "nltk==3.9.4", "Markdown==3.10.2", "beautifulsoup4==4.14.3", + "lxml==6.1.1", "pypandoc==1.17", "pandas==3.0.3", "openpyxl==3.1.5", diff --git a/uv.lock b/uv.lock index 2191f0d19c..8d8de336a5 100644 --- a/uv.lock +++ b/uv.lock @@ -3036,6 +3036,7 @@ dependencies = [ { name = "langchain-text-splitters" }, { name = "ldap3" }, { name = "loguru" }, + { name = "lxml" }, { name = "markdown" }, { name = "mcp" }, { name = "msoffcrypto-tool" }, @@ -3175,6 +3176,7 @@ requires-dist = [ { name = "langchain-text-splitters", specifier = "==1.1.2" }, { name = "ldap3", specifier = "==2.9.1" }, { name = "loguru", specifier = "==0.7.3" }, + { name = "lxml", specifier = "==6.1.1" }, { name = "mariadb", marker = "extra == 'mariadb'", specifier = "==1.1.14" }, { name = "markdown", specifier = "==3.10.2" }, { name = "mcp", specifier = "==1.27.2" },