AsyncHtmlLoader#

class langchain_community.document_loaders.async_html.AsyncHtmlLoader(
web_path: str | List[str],
header_template: dict | None = None,
verify_ssl: bool | None = True,
proxies: dict | None = None,
autoset_encoding: bool = True,
encoding: str | None = None,
default_parser: str = 'html.parser',
requests_per_second: int = 2,
requests_kwargs: Dict[str, Any] | None = None,
raise_for_status: bool = False,
ignore_load_errors: bool = False,
*,
preserve_order: bool = True,
trust_env: bool = False,
)[source]#

Load HTML asynchronously.

Initialize with a webpage path.

Methods

__init__(web_path[, header_template, ...])

Initialize with a webpage path.

alazy_load()

Lazy load text from the url(s) in web_path.

aload()

Load data into Document objects.

fetch_all(urls)

Fetch all urls concurrently with rate limiting.

lazy_load()

Lazy load text from the url(s) in web_path.

load()

Load data into Document objects.

load_and_split([text_splitter])

Load Documents and split into chunks.
Parameters:

  • web_path (str | List[str])

  • header_template (dict | None)

  • verify_ssl (bool | None)

  • proxies (dict | None)

  • autoset_encoding (bool)

  • encoding (str | None)

  • default_parser (str)

  • requests_per_second (int)

  • requests_kwargs (Dict[str, Any] | None)

  • raise_for_status (bool)

  • ignore_load_errors (bool)

  • preserve_order (bool)

  • trust_env (bool)

__init__(
web_path: str | List[str],
header_template: dict | None = None,
verify_ssl: bool | None = True,
proxies: dict | None = None,
autoset_encoding: bool = True,
encoding: str | None = None,
default_parser: str = 'html.parser',
requests_per_second: int = 2,
requests_kwargs: Dict[str, Any] | None = None,
raise_for_status: bool = False,
ignore_load_errors: bool = False,
*,
preserve_order: bool = True,
trust_env: bool = False,
)[source]#

Initialize with a webpage path.

Parameters:

  • web_path (str | List[str])

  • header_template (dict | None)

  • verify_ssl (bool | None)

  • proxies (dict | None)

  • autoset_encoding (bool)

  • encoding (str | None)

  • default_parser (str)

  • requests_per_second (int)

  • requests_kwargs (Dict[str, Any] | None)

  • raise_for_status (bool)

  • ignore_load_errors (bool)

  • preserve_order (bool)

  • trust_env (bool)

async alazy_load() → AsyncIterator[Document][source]#

Lazy load text from the url(s) in web_path.

Return type:

AsyncIterator[Document]

async aload() → list[Document]#

Load data into Document objects.

Return type:

list[Document]

async fetch_all(
urls: List[str],
) → List[str][source]#

Fetch all urls concurrently with rate limiting.

Parameters:

urls (List[str])

Return type:

List[str]

lazy_load() → Iterator[Document][source]#

Lazy load text from the url(s) in web_path.

Return type:

Iterator[Document]

load() → list[Document]#

Load data into Document objects.

Return type:

list[Document]

load_and_split(
text_splitter: TextSplitter | None = None,
) → list[Document]#

Load Documents and split into chunks. Chunks are returned as Documents.

Do not override this method. It should be considered to be deprecated!

Parameters:

text_splitter (Optional[TextSplitter]) – TextSplitter instance to use for splitting documents. Defaults to RecursiveCharacterTextSplitter.

Returns:

List of Documents.

Return type:

list[Document]

Examples using AsyncHtmlLoader