modify lstm policy

Author: quantumiracle

MUJOCO_LOG.TXT

@@ -0,0 +1,9 @@
+Wed Jan  8 11:15:37 2020
+ERROR: Expired activation key
+
+Wed Jan  8 11:16:43 2020
+ERROR: Expired activation key
+
+Wed Jan  8 11:20:15 2020
+ERROR: Expired activation key
+

POMDP/sac_v2_lstm.py

@@ -242,16 +242,16 @@ def plot(rewards):
                     next_state, reward, done, _ = env.step(action)
                     # env.render()       
 
-                if step>0:
+                if step == 0:
                     ini_hidden_in = hidden_in
                     ini_hidden_out = hidden_out
-                    episode_state.append(state)
-                    episode_action.append(action)
-                    episode_last_action.append(last_action)
-                    episode_reward.append(reward)
-                    episode_next_state.append(next_state)
-                    episode_done.append(done)  
-
+                episode_state.append(state)
+                episode_action.append(action)
+                episode_last_action.append(last_action)
+                episode_reward.append(reward)
+                episode_next_state.append(next_state)
+                episode_done.append(done) 
+                
                 state = next_state
                 last_action = action
                 frame_idx += 1

POMDP/td3_lstm.py

@@ -239,15 +239,15 @@ def plot(rewards):
                     next_state, reward, done, _ = env.step(action) 
                     # env.render()
 
-                if step>0:
+                if step == 0:
                     ini_hidden_in = hidden_in
                     ini_hidden_out = hidden_out
-                    episode_state.append(state)
-                    episode_action.append(action)
-                    episode_last_action.append(last_action)
-                    episode_reward.append(reward)
-                    episode_next_state.append(next_state)
-                    episode_done.append(done)  
+                episode_state.append(state)
+                episode_action.append(action)
+                episode_last_action.append(last_action)
+                episode_reward.append(reward)
+                episode_next_state.append(next_state)
+                episode_done.append(done) 
 
                 state = next_state
                 last_action = action

ppo_continuous.py

@@ -55,7 +55,7 @@
 
 #####################  hyper parameters  ####################
 
-ENV_NAME = 'Pendulum-v0'  # environment name HalfCheetah-v2 Pendulum-v0
+ENV_NAME = 'HalfCheetah-v2'  # environment name HalfCheetah-v2 Pendulum-v0
 RANDOMSEED = 2  # random seed
 
 EP_MAX = 1000  # total number of episodes for training

ppo_continuous_multiprocess.py

@@ -455,8 +455,6 @@ def main():
 
         ppo.save_model(MODEL_PATH)
 
-
-
     if args.test:
         ppo.load_model(MODEL_PATH)
         while True:

sac_v2_gru.py

@@ -262,15 +262,15 @@ def plot(rewards):
                     next_state, reward, done, _ = env.step(action)
                     env.render()       
 
-                if step>0:
+                if step == 0:
                     ini_hidden_in = hidden_in
                     ini_hidden_out = hidden_out
-                    episode_state.append(state)
-                    episode_action.append(action)
-                    episode_last_action.append(last_action)
-                    episode_reward.append(reward)
-                    episode_next_state.append(next_state)
-                    episode_done.append(done)  
+                episode_state.append(state)
+                episode_action.append(action)
+                episode_last_action.append(last_action)
+                episode_reward.append(reward)
+                episode_next_state.append(next_state)
+                episode_done.append(done) 
 
                 state = next_state
                 last_action = action

sac_v2_lstm.py

@@ -259,17 +259,17 @@ def plot(rewards):
                     next_state, reward, done, _ = env.step(action, SPARSE_REWARD, SCREEN_SHOT)
                 else:
                     next_state, reward, done, _ = env.step(action)
-                    env.render()       
+                    # env.render()       
 
-                if step>0:
+                if step == 0:
                     ini_hidden_in = hidden_in
                     ini_hidden_out = hidden_out
-                    episode_state.append(state)
-                    episode_action.append(action)
-                    episode_last_action.append(last_action)
-                    episode_reward.append(reward)
-                    episode_next_state.append(next_state)
-                    episode_done.append(done)  
+                episode_state.append(state)
+                episode_action.append(action)
+                episode_last_action.append(last_action)
+                episode_reward.append(reward)
+                episode_next_state.append(next_state)
+                episode_done.append(done) 
 
                 state = next_state
                 last_action = action

td3_lstm.py

@@ -248,16 +248,15 @@ def plot(rewards):
                 else:
                     next_state, reward, done, _ = env.step(action) 
                     # env.render()
-
-                if step>0:
+                if step == 0:
                     ini_hidden_in = hidden_in
                     ini_hidden_out = hidden_out
-                    episode_state.append(state)
-                    episode_action.append(action)
-                    episode_last_action.append(last_action)
-                    episode_reward.append(reward)
-                    episode_next_state.append(next_state)
-                    episode_done.append(done)  
+                episode_state.append(state)
+                episode_action.append(action)
+                episode_last_action.append(last_action)
+                episode_reward.append(reward)
+                episode_next_state.append(next_state)
+                episode_done.append(done)  
 
                 state = next_state
                 last_action = action