11importmatplotlib.pyplotasplt12importnumpyasnp13importtorch14importtorch.nnasnn1516fromlabml\_helpers.moduleimportModule

19classLabelSmoothingLoss(Module):

20def\_\_init\_\_(self,size:int,padding\_idx:int,smoothing:float=0.0):21super().\_\_init\_\_()22self.loss=nn.KLDivLoss(reduction='sum')23self.padding\_idx=padding\_idx24self.confidence=1.0-smoothing25self.smoothing=smoothing26self.size=size27self.true\_dist=None

29defforward(self,x:torch.Tensor,target:torch.Tensor):30assertx.shape[1]==self.size31true\_dist=x.clone()32true\_dist.fill\_(self.smoothing/(self.size-2))33true\_dist.scatter\_(1,target.unsqueeze(1),self.confidence)34true\_dist[:,self.padding\_idx]=035mask=torch.nonzero(target==self.padding\_idx,as\_tuple=False)36ifmask.dim()\>0:37true\_dist.index\_fill\_(0,mask.squeeze(),0.0)38self.true\_dist=true\_dist39returnself.loss(x,true\_dist.detach())

42def\_test\_label\_smoothing():43smooth\_loss=LabelSmoothingLoss(5,0,0.4)44predict=torch.tensor([[0,0.2,0.7,0.1,0],45[0,0.2,0.7,0.1,0],46[0,0.2,0.7,0.1,0]],dtype=torch.float)47\_=smooth\_loss(predict.log(),48torch.tensor([2,1,0],dtype=torch.long))

51plt.imshow(smooth\_loss.true\_dist)52plt.show()5354smooth\_loss=LabelSmoothingLoss(5,0,0.1)

56defloss\_sample(x):57d=x+3\*158predict2=torch.tensor([[0,x/d,1/d,1/d,1/d],59],dtype=torch.float)

61returnsmooth\_loss(predict2.log(),62torch.tensor([1],dtype=torch.long)).item()6364plt.plot(np.arange(1,100),[loss\_sample(x)forxinrange(1,100)])65plt.show()666768if\_\_name\_\_=='\_\_main\_\_':69\_test\_label\_smoothing()