def indexesFromSentence(lang, sentence): return [lang.word2index[word] for word in sentence.split(' ')] def tensorFromSentence(lang, sentence): indexes = indexesFromSentence(lang, sentence) indexes.append(EOS_token) return [torch.tensor](https://docs.pytorch.org/docs/stable/generated/torch.tensor.html#torch.tensor "torch.tensor")(indexes, dtype=[torch.long](https://docs.pytorch.org/docs/stable/tensor_attributes.html#torch.dtype "torch.dtype"), [device](https://docs.pytorch.org/docs/stable/tensor_attributes.html#torch.device "torch.device")=[device](https://docs.pytorch.org/docs/stable/tensor_attributes.html#torch.device "torch.device")).view(1, -1) def tensorsFromPair(pair): input_tensor = tensorFromSentence(input_lang, pair[0]) target_tensor = tensorFromSentence(output_lang, pair[1]) return (input_tensor, target_tensor) def get_dataloader(batch_size): input_lang, output_lang, pairs = prepareData('eng', 'fra', True) n = len(pairs) input_ids = np.zeros((n, MAX_LENGTH), dtype=np.int32) target_ids = np.zeros((n, MAX_LENGTH), dtype=np.int32) for idx, (inp, tgt) in enumerate(pairs): inp_ids = indexesFromSentence(input_lang, inp) tgt_ids = indexesFromSentence(output_lang, tgt) inp_ids.append(EOS_token) tgt_ids.append(EOS_token) input_ids[idx, :len(inp_ids)] = inp_ids target_ids[idx, :len(tgt_ids)] = tgt_ids train_data = [TensorDataset](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.TensorDataset "torch.utils.data.TensorDataset")(torch.LongTensor(input_ids).to([device](https://docs.pytorch.org/docs/stable/tensor_attributes.html#torch.device "torch.device")), torch.LongTensor(target_ids).to([device](https://docs.pytorch.org/docs/stable/tensor_attributes.html#torch.device "torch.device"))) train_sampler = [RandomSampler](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.RandomSampler "torch.utils.data.RandomSampler")(train_data) [train_dataloader](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader "torch.utils.data.DataLoader") = [DataLoader](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader "torch.utils.data.DataLoader")(train_data, sampler=train_sampler, batch_size=batch_size) return input_lang, output_lang, [train_dataloader](https://docs.pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader "torch.utils.data.DataLoader")