长兴县建设管理网站,莱芜金点子官网,工作室装修网站源码,php做的网站怎么运行PyTorch 读取大数据
数据量太大#xff0c;必须分批从磁盘加载#xff0c;下面是单机单卡的思路#xff1a;
from torch.utils.data import Dataset, DataLoader
import torchclass PretrainData(Dataset):def __init__(self):假设data是个数据量很大的文件#xff0c;每…PyTorch 读取大数据
数据量太大必须分批从磁盘加载下面是单机单卡的思路
from torch.utils.data import Dataset, DataLoader
import torchclass PretrainData(Dataset):def __init__(self):假设data是个数据量很大的文件每次只能从内存中加载3条数据后续可以把data放在odps_batch_data中改写成从文件中读数据self.meta_list []self.shift 0self.odps_batch 3self.data [[1, 1, 1], [2, 2, 2], [3, 3, 3], [4, 4, 4], [5, 5, 5], [6, 6, 6], [7, 7, 7], [8, 8, 8]]self.datalength len(self.data)def __len__(self):if len(self.meta_list) 0:return self.odps_batchelse:return len(self.meta_list)def __getitem__(self, idx):return self.meta_list[idx]def get_odps_batch_data(self):通过偏移量shift和大小odps_batch来从表中读数据if self.shift self.odps_batch self.datalength:self.meta_list torch.tensor(self.data[self.shift:self.shift self.odps_batch])else:self.meta_list torch.tensor(self.data[self.shift:])print(self.meta_list:, self.meta_list)if __name__ __main__:dataset PretrainData()dataloader DataLoader(dataset, batch_size2, shuffleTrue, drop_lastFalse)for epoch in range(3):for shift in range(dataset.datalength // dataset.odps_batch 1):dataset.shift shift * dataset.odps_batchdataloader.dataset.get_odps_batch_data()print(len(dataloader))for data in dataloader:print(epoch:, epoch)print(data:, data)user data[:, 0]item data[:, 1]click data[:, 2]print(user, item, click)
