11fromlabmlimportexperiment12fromlabml.configsimportcalculate13fromlabml\_nn.experiments.arithmetic\_datasetimportArithmeticAutoregression14fromlabml\_nn.transformersimportTransformerConfigs15fromlabml\_nn.transformers.rope.experimentimportConfigsasRoPEConfigs

18classConfigs(RoPEConfigs,ArithmeticAutoregression):

26pass

29def\_rotary\_value\_pe\_mha(c:TransformerConfigs):

33fromlabml\_nn.transformers.rope.value\_peimportRotaryValuePEMultiHeadAttention34returnRotaryValuePEMultiHeadAttention(c.n\_heads,c.d\_model,1.,1.)

38calculate(TransformerConfigs.encoder\_attn,'rotary\_value',\_rotary\_value\_pe\_mha)39calculate(TransformerConfigs.decoder\_attn,'rotary\_value',\_rotary\_value\_pe\_mha)40calculate(TransformerConfigs.decoder\_mem\_attn,'rotary\_value',\_rotary\_value\_pe\_mha)

43defmain():

45experiment.create(name="roper\_addition",comment="rotary value 7",writers={'screen','labml'})

47conf=Configs()

49experiment.configs(conf,{50'max\_digits':7,

53'transformer.src\_embed':'no\_pos',54'transformer.tgt\_embed':'no\_pos',

57'transformer.encoder\_attn':'rotary\_value',

62'model':'rotary\_pe\_transformer',

65'seq\_len':512,

67'epochs':20,

69'batch\_size':16,

72'd\_model':128,73'transformer.ffn.d\_ff':512,74'transformer.n\_heads':4,75'transformer.dropout':0.0,

78'optimizer.optimizer':'Adam',79'optimizer.learning\_rate':2.5e-4,80})

83experiment.add\_pytorch\_models({'model':conf.model})

86withexperiment.start():

88conf.run()

92if\_\_name\_\_=='\_\_main\_\_':93main()