modify ppo std

Author: quantumiracle

ppo_continuous.py

@@ -55,7 +55,7 @@
 
 #####################  hyper parameters  ####################
 
-ENV_NAME = 'HalfCheetah-v2'  # environment name HalfCheetah-v2 Pendulum-v0
+ENV_NAME = 'Pendulum-v0'  # environment name HalfCheetah-v2 Pendulum-v0
 RANDOMSEED = 2  # random seed
 
 EP_MAX = 1000  # total number of episodes for training
@@ -74,6 +74,20 @@
 
 ###############################  PPO  ####################################
 
+class AddBias(nn.Module):
+    def __init__(self, bias):
+        super(AddBias, self).__init__()
+        self._bias = nn.Parameter(bias.unsqueeze(1))
+
+    def forward(self, x):
+        if x.dim() == 2:
+            bias = self._bias.t().view(1, -1)
+        else:
+            bias = self._bias.t().view(1, -1, 1, 1)
+
+        return x + bias
+
+
 class ValueNetwork(nn.Module):
     def __init__(self, state_dim, hidden_dim, init_w=3e-3):
         super(ValueNetwork, self).__init__()
@@ -106,12 +120,10 @@ def __init__(self, num_inputs, num_actions, hidden_dim, action_range=1., init_w=
         # self.linear4 = nn.Linear(hidden_dim, hidden_dim)
 
         self.mean_linear = nn.Linear(hidden_dim, num_actions)
-        # self.mean_linear.weight.data.uniform_(-init_w, init_w)
-        # self.mean_linear.bias.data.uniform_(-init_w, init_w)
-        
-        self.log_std_linear = nn.Linear(hidden_dim, num_actions)
-        # self.log_std_linear.weight.data.uniform_(-init_w, init_w)
-        # self.log_std_linear.bias.data.uniform_(-init_w, init_w)
+        # implementation 1
+        # self.log_std_linear = nn.Linear(hidden_dim, num_actions)
+        # # implementation 2: not dependent on latent features, reference:https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail/blob/master/a2c_ppo_acktr/distributions.py
+        self.log_std = AddBias(torch.zeros(num_actions))  
 
         self.num_actions = num_actions
         self.action_range = action_range
@@ -123,9 +135,17 @@ def forward(self, state):
         # x = F.relu(self.linear4(x))
 
         mean    = self.action_range * F.tanh(self.mean_linear(x))
-        log_std = self.log_std_linear(x)
-        log_std = torch.clamp(log_std, self.log_std_min, self.log_std_max)
-        
+
+        # implementation 1
+        # log_std = self.log_std_linear(x)
+        # log_std = torch.clamp(log_std, self.log_std_min, self.log_std_max)
+    
+        # implementation 2
+        zeros = torch.zeros(mean.size())
+        if state.is_cuda:
+            zeros = zeros.cuda()
+        log_std = self.log_std(zeros)
+
         return mean, log_std
 
     def get_action(self, state, deterministic=False):

ppo_continuous_multiprocess.py

@@ -64,14 +64,29 @@
 EPS = 1e-8   # numerical residual
 MODEL_PATH = 'model/ppo_multi'
 NUM_WORKERS=2  # or: mp.cpu_count()
-ACTION_RANGE = 2.  # if unnormalized, normalized action range should be 1.
+ACTION_RANGE = 1.  # if unnormalized, normalized action range should be 1.
 METHOD = [
     dict(name='kl_pen', kl_target=0.01, lam=0.5),  # KL penalty
     dict(name='clip', epsilon=0.2),  # Clipped surrogate objective, find this is better
 ][0]  # choose the method for optimization
 
 ###############################  PPO  ####################################
 
+
+class AddBias(nn.Module):
+    def __init__(self, bias):
+        super(AddBias, self).__init__()
+        self._bias = nn.Parameter(bias.unsqueeze(1))
+
+    def forward(self, x):
+        if x.dim() == 2:
+            bias = self._bias.t().view(1, -1)
+        else:
+            bias = self._bias.t().view(1, -1, 1, 1)
+
+        return x + bias
+
+
 class ValueNetwork(nn.Module):
     def __init__(self, state_dim, hidden_dim, init_w=3e-3):
         super(ValueNetwork, self).__init__()
@@ -104,12 +119,10 @@ def __init__(self, num_inputs, num_actions, hidden_dim, action_range=1., init_w=
         # self.linear4 = nn.Linear(hidden_dim, hidden_dim)
 
         self.mean_linear = nn.Linear(hidden_dim, num_actions)
-        self.mean_linear.weight.data.uniform_(-init_w, init_w)
-        self.mean_linear.bias.data.uniform_(-init_w, init_w)
-        
-        self.log_std_linear = nn.Linear(hidden_dim, num_actions)
-        self.log_std_linear.weight.data.uniform_(-init_w, init_w)
-        self.log_std_linear.bias.data.uniform_(-init_w, init_w)
+        # implementation 1
+        # self.log_std_linear = nn.Linear(hidden_dim, num_actions)
+        # # implementation 2: not dependent on latent features, reference:https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail/blob/master/a2c_ppo_acktr/distributions.py
+        self.log_std = AddBias(torch.zeros(num_actions))  
 
         self.num_actions = num_actions
         self.action_range = action_range
@@ -122,8 +135,15 @@ def forward(self, state):
         # x = F.relu(self.linear4(x))
 
         mean    = self.action_range * F.tanh(self.mean_linear(x))
-        log_std = self.log_std_linear(x)
-        log_std = torch.clamp(log_std, self.log_std_min, self.log_std_max)
+        # implementation 1
+        # log_std = self.log_std_linear(x)
+        # log_std = torch.clamp(log_std, self.log_std_min, self.log_std_max)
+    
+        # implementation 2
+        zeros = torch.zeros(mean.size())
+        if state.is_cuda:
+            zeros = zeros.cuda()
+        log_std = self.log_std(zeros)
 
         return mean, log_std
 
@@ -396,7 +416,7 @@ def main():
     np.random.seed(RANDOMSEED)
     torch.manual_seed(RANDOMSEED)
 
-    env = gym.make(ENV_NAME).unwrapped
+    env = NormalizedActions(gym.make(ENV_NAME).unwrapped)
     state_dim = env.observation_space.shape[0]
     action_dim = env.action_space.shape[0]