AZLyricsLoader#

class langchain_community.document_loaders.azlyrics.AZLyricsLoader(
web_path: str | Sequence[str] = '',
header_template: dict | None = None,
verify_ssl: bool = True,
proxies: dict | None = None,
continue_on_failure: bool = False,
autoset_encoding: bool = True,
encoding: str | None = None,
web_paths: Sequence[str] = (),
requests_per_second: int = 2,
default_parser: str = 'html.parser',
requests_kwargs: Dict[str, Any] | None = None,
raise_for_status: bool = False,
bs_get_text_kwargs: Dict[str, Any] | None = None,
bs_kwargs: Dict[str, Any] | None = None,
session: Any = None,
*,
show_progress: bool = True,
trust_env: bool = False,
)[source]#

Load AZLyrics webpages.

Initialize loader.

Parameters:

  • web_paths (Sequence[str]) – Web paths to load from.

  • requests_per_second (int) – Max number of concurrent requests to make.

  • default_parser (str) – Default parser to use for BeautifulSoup.

  • requests_kwargs (Dict[str, Any] | None) – kwargs for requests

  • raise_for_status (bool) – Raise an exception if http status code denotes an error.

  • bs_get_text_kwargs (Dict[str, Any] | None) – kwargs for beatifulsoup4 get_text

  • bs_kwargs (Dict[str, Any] | None) – kwargs for beatifulsoup4 web page parsing

  • show_progress (bool) – Show progress bar when loading pages.

  • trust_env (bool) – set to True if using proxy to make web requests, for example using http(s)_proxy environment variables. Defaults to False.

  • web_path (str | Sequence[str])

  • header_template (dict | None)

  • verify_ssl (bool)

  • proxies (dict | None)

  • continue_on_failure (bool)

  • autoset_encoding (bool)

  • encoding (str | None)

  • session (Any)

Attributes

web_path

Methods

__init__([web_path, header_template, ...])

Initialize loader.

alazy_load()

Async lazy load text from the url(s) in web_path.

aload()

ascrape_all(urls[, parser])

Async fetch all urls, then return soups for all results.

fetch_all(urls)

Fetch all urls concurrently with rate limiting.

lazy_load()

Lazy load text from the url(s) in web_path.

load()

Load webpages into Documents.

load_and_split([text_splitter])

Load Documents and split into chunks.

scrape([parser])

Scrape data from webpage and return it in BeautifulSoup format.

scrape_all(urls[, parser])

Fetch all urls, then return soups for all results.
__init__(
web_path: str | Sequence[str] = '',
header_template: dict | None = None,
verify_ssl: bool = True,
proxies: dict | None = None,
continue_on_failure: bool = False,
autoset_encoding: bool = True,
encoding: str | None = None,
web_paths: Sequence[str] = (),
requests_per_second: int = 2,
default_parser: str = 'html.parser',
requests_kwargs: Dict[str, Any] | None = None,
raise_for_status: bool = False,
bs_get_text_kwargs: Dict[str, Any] | None = None,
bs_kwargs: Dict[str, Any] | None = None,
session: Any = None,
*,
show_progress: bool = True,
trust_env: bool = False,
) → None#

Initialize loader.

Parameters:

  • web_paths (Sequence[str]) – Web paths to load from.

  • requests_per_second (int) – Max number of concurrent requests to make.

  • default_parser (str) – Default parser to use for BeautifulSoup.

  • requests_kwargs (Dict[str, Any] | None) – kwargs for requests

  • raise_for_status (bool) – Raise an exception if http status code denotes an error.

  • bs_get_text_kwargs (Dict[str, Any] | None) – kwargs for beatifulsoup4 get_text

  • bs_kwargs (Dict[str, Any] | None) – kwargs for beatifulsoup4 web page parsing

  • show_progress (bool) – Show progress bar when loading pages.

  • trust_env (bool) – set to True if using proxy to make web requests, for example using http(s)_proxy environment variables. Defaults to False.

  • web_path (str | Sequence[str])

  • header_template (dict | None)

  • verify_ssl (bool)

  • proxies (dict | None)

  • continue_on_failure (bool)

  • autoset_encoding (bool)

  • encoding (str | None)

  • session (Any)

Return type:

None

async alazy_load() → AsyncIterator[Document]#

Async lazy load text from the url(s) in web_path.

Return type:

AsyncIterator[Document]

aload() → List[Document]#

Deprecated since version 0.3.14: See API reference for updated usage: https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.web_base.WebBaseLoader.html It will not be removed until langchain-community==1.0.

Load text from the urls in web_path async into Documents.

Return type:

List[Document]

async ascrape_all(
urls: List[str],
parser: str | None = None,
) → List[Any]#

Async fetch all urls, then return soups for all results.

Parameters:

  • urls (List[str])

  • parser (str | None)

Return type:

List[Any]

async fetch_all(
urls: List[str],
) → Any#

Fetch all urls concurrently with rate limiting.

Parameters:

urls (List[str])

Return type:

Any

lazy_load() → Iterator[Document]#

Lazy load text from the url(s) in web_path.

Return type:

Iterator[Document]

load() → List[Document][source]#

Load webpages into Documents.

Return type:

List[Document]

load_and_split(
text_splitter: TextSplitter | None = None,
) → list[Document]#

Load Documents and split into chunks. Chunks are returned as Documents.

Do not override this method. It should be considered to be deprecated!

Parameters:

text_splitter (Optional[TextSplitter]) – TextSplitter instance to use for splitting documents. Defaults to RecursiveCharacterTextSplitter.

Returns:

List of Documents.

Return type:

list[Document]

scrape(
parser: str | None = None,
) → Any#

Scrape data from webpage and return it in BeautifulSoup format.

Parameters:

parser (str | None)

Return type:

Any

scrape_all(
urls: List[str],
parser: str | None = None,
) → List[Any]#

Fetch all urls, then return soups for all results.

Parameters:

  • urls (List[str])

  • parser (str | None)

Return type:

List[Any]

Examples using AZLyricsLoader