Source code for langchain_community.document_loaders.merge

from typing import AsyncIterator, Iterator, List

from langchain_core.documents import Document

from langchain_community.document_loaders.base import BaseLoader


[docs]class MergedDataLoader(BaseLoader): """Merge documents from a list of loaders"""
[docs] def __init__(self, loaders: List): """Initialize with a list of loaders""" self.loaders = loaders
[docs] def lazy_load(self) -> Iterator[Document]: """Lazy load docs from each individual loader.""" for loader in self.loaders: # Check if lazy_load is implemented try: data = loader.lazy_load() except NotImplementedError: data = loader.load() for document in data: yield document
[docs] async def alazy_load(self) -> AsyncIterator[Document]: """Lazy load docs from each individual loader.""" for loader in self.loaders: async for document in loader.alazy_load(): yield document