Dataloaders

get(name=None, dataset_params=None, dataloader_params=None, dataset=None)

Get DataLoader of the recipe-configured dataset defined by name in ALL_DATALOADERS.

Parameters:

Name	Type	Description	Default
name	str	dataset name in ALL_DATALOADERS.	None
dataset_params	Dict	dataset params that override the yaml configured defaults, then passed to the dataset_cls.init.	None
dataloader_params	Dict	DataLoader params that override the yaml configured defaults, then passed to the DataLoader.init	None
dataset	torch.utils.data.Dataset	torch.utils.data.Dataset to be used instead of passing "name" (i.e for external dataset objects).	None

Returns:

Type	Description
DataLoader	initialized DataLoader.

Source code in training/dataloaders/dataloaders.py

def get(name: str = None, dataset_params: Dict = None, dataloader_params: Dict = None, dataset: torch.utils.data.Dataset = None) -> DataLoader:
    """
    Get DataLoader of the recipe-configured dataset defined by name in ALL_DATALOADERS.

    :param name: dataset name in ALL_DATALOADERS.
    :param dataset_params: dataset params that override the yaml configured defaults, then passed to the dataset_cls.__init__.
    :param dataloader_params: DataLoader params that override the yaml configured defaults, then passed to the DataLoader.__init__
    :param dataset: torch.utils.data.Dataset to be used instead of passing "name" (i.e for external dataset objects).
    :return: initialized DataLoader.
    """
    if dataset is not None:
        if name or dataset_params:
            raise ValueError("'name' and 'dataset_params' cannot be passed with initialized dataset.")

    dataset_str = get_param(dataloader_params, "dataset")

    if dataset_str:
        if name or dataset:
            raise ValueError("'name' and 'datasets' cannot be passed when 'dataset' arg dataloader_params is used as well.")
        if dataset_params is not None:
            dataset = DatasetsFactory().get(conf={dataset_str: dataset_params})
        else:
            dataset = DatasetsFactory().get(conf=dataset_str)
        _ = dataloader_params.pop("dataset")

    if dataset is not None:
        dataloader_params = _process_sampler_params(dataloader_params, dataset, {})
        dataloader = DataLoader(dataset=dataset, **dataloader_params)
    elif name not in ALL_DATALOADERS.keys():
        raise ValueError("Unsupported dataloader: " + str(name))
    else:
        dataloader_cls = ALL_DATALOADERS[name]
        dataloader = dataloader_cls(dataset_params=dataset_params, dataloader_params=dataloader_params)

    return dataloader

get_data_loader(config_name, dataset_cls, train, dataset_params=None, dataloader_params=None)

Class for creating dataloaders for taking defaults from yaml files in src/super_gradients/recipes.

Parameters:

Name	Type	Description	Default
config_name	str	yaml config filename of dataset_params in recipes (for example coco_detection_dataset_params).	required
dataset_cls	object	torch dataset uninitialized class.	required
train	bool	controls whether to take cfg.train_dataloader_params or cfg.valid_dataloader_params as defaults for the dataset constructor and cfg.train_dataset_params or cfg.valid_dataset_params as defaults for DataLoader contructor.	required
dataset_params	Mapping	dataset params that override the yaml configured defaults, then passed to the dataset_cls.init.	None
dataloader_params	Mapping	DataLoader params that override the yaml configured defaults, then passed to the DataLoader.init	None

Returns:

Type	Description
DataLoader	DataLoader

Source code in training/dataloaders/dataloaders.py

def get_data_loader(config_name: str, dataset_cls: object, train: bool, dataset_params: Mapping = None, dataloader_params: Mapping = None) -> DataLoader:
    """
    Class for creating dataloaders for taking defaults from yaml files in src/super_gradients/recipes.

    :param config_name: yaml config filename of dataset_params in recipes (for example coco_detection_dataset_params).
    :param dataset_cls: torch dataset uninitialized class.
    :param train: controls whether to take
        cfg.train_dataloader_params or cfg.valid_dataloader_params as defaults for the dataset constructor
     and
        cfg.train_dataset_params or cfg.valid_dataset_params as defaults for DataLoader contructor.

    :param dataset_params: dataset params that override the yaml configured defaults, then passed to the dataset_cls.__init__.
    :param dataloader_params: DataLoader params that override the yaml configured defaults, then passed to the DataLoader.__init__
    :return: DataLoader
    """
    if dataloader_params is None:
        dataloader_params = dict()
    if dataset_params is None:
        dataset_params = dict()

    cfg = load_dataset_params(config_name=config_name)

    dataset_params = _process_dataset_params(cfg, dataset_params, train)

    local_rank = get_local_rank()
    with wait_for_the_master(local_rank):
        dataset = dataset_cls(**dataset_params)
        if not hasattr(dataset, "dataset_params"):
            dataset.dataset_params = dataset_params

    dataloader_params = _process_dataloader_params(cfg, dataloader_params, dataset, train)

    dataloader = DataLoader(dataset=dataset, **dataloader_params)
    dataloader.dataloader_params = dataloader_params
    return dataloader