9importmultiprocessing10importmultiprocessing.connection1112importcv213importgym14importnumpyasnp

17classGame:

38def\_\_init\_\_(self,seed:int):

40self.env=gym.make('BreakoutNoFrameskip-v4')41self.env.seed(seed)

44self.obs\_4=np.zeros((4,84,84))

47self.obs\_2\_max=np.zeros((2,84,84))

50self.rewards=[]

52self.lives=0

54defstep(self,action):

66reward=0.67done=None

70foriinrange(4):

72obs,r,done,info=self.env.step(action)7374ifi\>=2:75self.obs\_2\_max[i%2]=self.\_process\_obs(obs)7677reward+=r

80lives=self.env.unwrapped.ale.lives()

82iflives\<self.lives:83done=True84break

87self.rewards.append(reward)8889ifdone:

91episode\_info={"reward":sum(self.rewards),"length":len(self.rewards)}92self.reset()93else:94episode\_info=None

97obs=self.obs\_2\_max.max(axis=0)

100self.obs\_4=np.roll(self.obs\_4,shift=-1,axis=0)101self.obs\_4[-1]=obs102103returnself.obs\_4,reward,done,episode\_info

105defreset(self):

112obs=self.env.reset()

115obs=self.\_process\_obs(obs)116foriinrange(4):117self.obs\_4[i]=obs118self.rewards=[]119120self.lives=self.env.unwrapped.ale.lives()121122returnself.obs\_4

124@staticmethod125def\_process\_obs(obs):

130obs=cv2.cvtColor(obs,cv2.COLOR\_RGB2GRAY)131obs=cv2.resize(obs,(84,84),interpolation=cv2.INTER\_AREA)132returnobs

135defworker\_process(remote:multiprocessing.connection.Connection,seed:int):

143game=Game(seed)

146whileTrue:147cmd,data=remote.recv()148ifcmd=="step":149remote.send(game.step(data))150elifcmd=="reset":151remote.send(game.reset())152elifcmd=="close":153remote.close()154break155else:156raiseNotImplementedError

159classWorker:

164def\_\_init\_\_(self,seed):165self.child,parent=multiprocessing.Pipe()166self.process=multiprocessing.Process(target=worker\_process,args=(parent,seed))167self.process.start()