郑州免费网站建设哪家好,wordpress 获取父分类,国外用什么做网站,建设企业网站怎么样使用pytorch自定义DataSet#xff0c;以加载图像数据集为例#xff0c;实现一些骚操作
总共分为四步
构造一个my_dataset类#xff0c;继承自torch.utils.data.Dataset重写__getitem__ 和__len__ 类函数建立两个函数find_classes、has_file_allowed_extension#xff0c;…使用pytorch自定义DataSet以加载图像数据集为例实现一些骚操作
总共分为四步
构造一个my_dataset类继承自torch.utils.data.Dataset重写__getitem__ 和__len__ 类函数建立两个函数find_classes、has_file_allowed_extension直接从这copy过去建立my_make_dataset函数用来构造(path,lable)对
一、构造一个my_dataset类继承自torch.utils.data.Dataset
二、 重写__getitem__ 和__len__ 类函数
要构造Dataset的子类就必须要实现两个方法
getitem_(self, index)根据index来返回数据集中标号为index的元素及其标签。len_(self)返回数据集的长度。
class my_dataset(Dataset):def __init__(self,root_original, root_cdtfed, transformNone):super(my_dataset, self).__init__()self.transform transformself.root_original root_originalself.root_cdtfed root_cdtfedself.original_imgs []self.cdtfed_imgs []#add (img_path, label) to listsself.original_imgs my_make_dataset(root_original, class_to_idxNone, extensions(.jpg, .png), is_valid_fileNone)self.cdtfed_imgs my_make_dataset(root_original, class_to_idxNone, extensions(.jpg, .png), is_valid_fileNone)# super(my_dataset, self).__init__()def __getitem__(self, index): #这个方法是必须要有的用于按照索引读取每个元素的具体内容fn1, label1 self.original_imgs[index] #fn是图片path #fn和label分别获得imgs[index]也即是刚才每行中word[0]和word[1]的信息fn2, label2 self.cdtfed_imgs[index]img1 Image.open(fn1).convert(RGB) #按照path读入图片from PIL import Image # 按照路径读取图片img2 Image.open(fn2).convert(RGB) #按照path读入图片from PIL import Image # 按照路径读取图片if self.transform is not None:img1 self.transform(img1) #是否进行transformimg2 self.transform(img2) #是否进行transformimg_list [img1, img2]label label1name fn1return img_list,label,name #return很关键return回哪些内容那么我们在训练时循环读取每个batch时就能获得哪些内容def __len__(self): #这个函数也必须要写它返回的是数据集的长度也就是多少张图片要和loader的长度作区分return len(self.original_imgs)
三、建立两个函数find_classes、has_file_allowed_extension直接从这copy过去
def find_classes(directory: str) - Tuple[List[str], Dict[str, int]]:Finds the class folders in a dataset.See :class:DatasetFolder for details.classes sorted(entry.name for entry in os.scandir(directory) if entry.is_dir())if not classes:raise FileNotFoundError(fCouldnt find any class folder in {directory}.)class_to_idx {cls_name: i for i, cls_name in enumerate(classes)}return classes, class_to_idxdef has_file_allowed_extension(filename: str, extensions: Tuple[str, ...]) - bool:Checks if a file is an allowed extension.Args:filename (string): path to a fileextensions (tuple of strings): extensions to consider (lowercase)Returns:bool: True if the filename ends with one of given extensionsreturn filename.lower().endswith(extensions)建立my_make_dataset函数用来构造(path,lable)对
def my_make_dataset(directory: str,class_to_idx: Optional[Dict[str, int]] None,extensions: Optional[Tuple[str, ...]] None,is_valid_file: Optional[Callable[[str], bool]] None,
) - List[Tuple[str, int]]:Generates a list of samples of a form (path_to_sample, class).See :class:DatasetFolder for details.Note: The class_to_idx parameter is here optional and will use the logic of the find_classes functionby default.directory os.path.expanduser(directory)if class_to_idx is None:_, class_to_idx find_classes(directory)elif not class_to_idx:raise ValueError(class_to_index must have at least one entry to collect any samples.)both_none extensions is None and is_valid_file is Noneboth_something extensions is not None and is_valid_file is not Noneif both_none or both_something:raise ValueError(Both extensions and is_valid_file cannot be None or not None at the same time)if extensions is not None:def is_valid_file(x: str) - bool:return has_file_allowed_extension(x, cast(Tuple[str, ...], extensions))is_valid_file cast(Callable[[str], bool], is_valid_file)instances []available_classes set()for target_class in sorted(class_to_idx.keys()):class_index class_to_idx[target_class]target_dir os.path.join(directory, target_class)if not os.path.isdir(target_dir):continuefor root, _, fnames in sorted(os.walk(target_dir, followlinksTrue)):for fname in sorted(fnames):if is_valid_file(fname):path os.path.join(root, fname)# item path, [int(cl) for cl in target_class.split(_)]item path, target_classinstances.append(item)if target_class not in available_classes:available_classes.add(target_class)empty_classes set(class_to_idx.keys()) - available_classesif empty_classes:msg fFound no valid file for the classes {, .join(sorted(empty_classes))}. if extensions is not None:msg fSupported extensions are: {, .join(extensions)}raise FileNotFoundError(msg)return instances #instance:[item:(path, int(class_name)), ]附录完整代码
我这里传入两个root_dir因为我要用一个dataset加载两个数据集分别放在data1和data2里
class my_dataset(Dataset):def __init__(self,root_original, root_cdtfed, transformNone):super(my_dataset, self).__init__()self.transform transformself.root_original root_originalself.root_cdtfed root_cdtfedself.original_imgs []self.cdtfed_imgs []#add (img_path, label) to listsself.original_imgs my_make_dataset(root_original, class_to_idxNone, extensions(.jpg, .png), is_valid_fileNone)self.cdtfed_imgs my_make_dataset(root_original, class_to_idxNone, extensions(.jpg, .png), is_valid_fileNone)# super(my_dataset, self).__init__()def __getitem__(self, index): #这个方法是必须要有的用于按照索引读取每个元素的具体内容fn1, label1 self.original_imgs[index] #fn是图片path #fn和label分别获得imgs[index]也即是刚才每行中word[0]和word[1]的信息fn2, label2 self.cdtfed_imgs[index]img1 Image.open(fn1).convert(RGB) #按照path读入图片from PIL import Image # 按照路径读取图片img2 Image.open(fn2).convert(RGB) #按照path读入图片from PIL import Image # 按照路径读取图片if self.transform is not None:img1 self.transform(img1) #是否进行transformimg2 self.transform(img2) #是否进行transformimg_list [img1, img2]label label1name fn1return img_list,label,name #return很关键return回哪些内容那么我们在训练时循环读取每个batch时就能获得哪些内容def __len__(self): #这个函数也必须要写它返回的是数据集的长度也就是多少张图片要和loader的长度作区分return len(self.original_imgs)def find_classes(directory: str) - Tuple[List[str], Dict[str, int]]:Finds the class folders in a dataset.See :class:DatasetFolder for details.classes sorted(entry.name for entry in os.scandir(directory) if entry.is_dir())if not classes:raise FileNotFoundError(fCouldnt find any class folder in {directory}.)class_to_idx {cls_name: i for i, cls_name in enumerate(classes)}return classes, class_to_idxdef has_file_allowed_extension(filename: str, extensions: Tuple[str, ...]) - bool:Checks if a file is an allowed extension.Args:filename (string): path to a fileextensions (tuple of strings): extensions to consider (lowercase)Returns:bool: True if the filename ends with one of given extensionsreturn filename.lower().endswith(extensions)def my_make_dataset(directory: str,class_to_idx: Optional[Dict[str, int]] None,extensions: Optional[Tuple[str, ...]] None,is_valid_file: Optional[Callable[[str], bool]] None,
) - List[Tuple[str, int]]:Generates a list of samples of a form (path_to_sample, class).See :class:DatasetFolder for details.Note: The class_to_idx parameter is here optional and will use the logic of the find_classes functionby default.directory os.path.expanduser(directory)if class_to_idx is None:_, class_to_idx find_classes(directory)elif not class_to_idx:raise ValueError(class_to_index must have at least one entry to collect any samples.)both_none extensions is None and is_valid_file is Noneboth_something extensions is not None and is_valid_file is not Noneif both_none or both_something:raise ValueError(Both extensions and is_valid_file cannot be None or not None at the same time)if extensions is not None:def is_valid_file(x: str) - bool:return has_file_allowed_extension(x, cast(Tuple[str, ...], extensions))is_valid_file cast(Callable[[str], bool], is_valid_file)instances []available_classes set()for target_class in sorted(class_to_idx.keys()):class_index class_to_idx[target_class]target_dir os.path.join(directory, target_class)if not os.path.isdir(target_dir):continuefor root, _, fnames in sorted(os.walk(target_dir, followlinksTrue)):for fname in sorted(fnames):if is_valid_file(fname):path os.path.join(root, fname)# item path, [int(cl) for cl in target_class.split(_)]item path, target_classinstances.append(item)if target_class not in available_classes:available_classes.add(target_class)empty_classes set(class_to_idx.keys()) - available_classesif empty_classes:msg fFound no valid file for the classes {, .join(sorted(empty_classes))}. if extensions is not None:msg fSupported extensions are: {, .join(extensions)}raise FileNotFoundError(msg)return instances #instance:[item:(path, int(class_name)), ]
