class Encoder(nn.Module):
"""The base encoder interface for the encoder-decoder architecture."""
def __init__(self):
super().__init__()
# Later there can be additional arguments (e.g., length excluding padding)
def forward(self, X, *args):
raise NotImplementedErrorThe encoder-decoder abstraction
The state is the only channel. Swap the encoder and the same shape becomes Whisper (speech), image captioning, or a multimodal front-end.
class Decoder(nn.Module):
"""The base decoder interface for the encoder-decoder architecture."""
def __init__(self):
super().__init__()
def init_state(self, enc_all_outputs, *args):
raise NotImplementedError
def forward(self, X, state):
raise NotImplementedErrorMost sentences are short
Which is why a small num_steps and a masked loss pay off:
src_ids = [data.tokenizer.encode(s) for s in data.src_sents]
tgt_ids = [data.tokenizer.encode(s) for s in data.tgt_sents]
show_list_len_pair_hist(['source', 'target'], '# tokens per sentence',
'count', src_ids, tgt_ids);
The unrolled model
class Seq2Seq(d2l.EncoderDecoder):
"""The RNN encoder-decoder for sequence-to-sequence learning."""
def __init__(self, encoder, decoder, tgt_pad, lr):
super().__init__(encoder, decoder)
self.save_hyperparameters()
def validation_step(self, batch):
Y_hat = self(*batch[:-1])
self.plot('loss', self.loss(Y_hat, batch[-1]), train=False)
def configure_optimizers(self):
return torch.optim.Adam(self.parameters(), lr=self.lr)Training
2-layer GRU, width 256, dropout 0.2, Adam lr 0.005, clip 1, 30 epochs:
data = d2l.MTFraEng(batch_size=128)
embed_size, num_hiddens, num_layers, dropout = 256, 256, 2, 0.2
encoder = Seq2SeqEncoder(
len(data.tokenizer), embed_size, num_hiddens, num_layers, dropout)
decoder = Seq2SeqDecoder(
len(data.tokenizer), embed_size, num_hiddens, num_layers, dropout)
model = Seq2Seq(encoder, decoder, tgt_pad=data.tokenizer.pad, lr=0.005)
trainer = d2l.Trainer(max_epochs=30, gradient_clip_val=1, num_gpus=1)
trainer.fit(model, data)
The fixed-vector bottleneck
Everything about the source crosses one vector \mathbf{c}. chrF falls as the source grows:
import numpy as np
buckets = collections.defaultdict(list)
for s, t, p in zip(h_src, h_tgt, h_preds):
L = len(data.tokenizer.encode(s))
buckets[L].append(chrf(to_text(p, data.tokenizer), t))
xs = sorted(L for L in buckets if len(buckets[L]) >= 8 and L <= data.num_steps)
ys = [np.mean(buckets[L]) for L in xs]
d2l.set_figsize()
d2l.plt.plot(xs, ys, 'o-')
d2l.plt.xlabel('source length (tokens)')
d2l.plt.ylabel('mean chrF')
d2l.plt.grid(True);
Two escapes, both already taken: attention (look back at all encoder states — where the attention chapters began) or a better recurrent state (the state space models you built).