INNER CODE UNIT · Python

LEARNING_RATE

lucidrains/x-transformers · train_copy.py:10

LEARNING_RATE = 3e-4
GENERATE_EVERY  = 100
NUM_TOKENS = 16 + 2
ENC_SEQ_LEN = 32
DEC_SEQ_LEN = 64 + 1
DEVICE = default_device()

# helpers

def cycle():
    while True:
        prefix = torch.ones((BATCH_SIZE, 1)).long().to(DEVICE)
        src = torch.randint(2, NUM_TOKENS, (BATCH_SIZE, ENC_SEQ_LEN)).long().to(DEVICE)
        tgt = torch.cat((prefix, src, src), 1)
        src_mask = torch.ones(BATCH_SIZE, src.shape[1]).bool().to(DEVICE)
        yield (src, tgt, src_mask)

# instantiate model

View source record →