Merge pull request udacity#12 from udacity/smartcab

smartcab: Penalties for violations/bad moves, simulation speed-ups, pygame now optional

projects/smartcab/smartcab/agent.py

@@ -41,11 +41,15 @@ def run():
     # Set up environment and agent
     e = Environment()  # create environment (also adds some dummy traffic)
     a = e.create_agent(LearningAgent)  # create agent
-    e.set_primary_agent(a, enforce_deadline=False)  # set agent to track
+    e.set_primary_agent(a, enforce_deadline=True)  # specify agent to track
+    # NOTE: You can set enforce_deadline=False while debugging to allow longer trials
 
     # Now simulate it
-    sim = Simulator(e, update_delay=1.0)  # reduce update_delay or add 'display=False' to speed up simulation
-    sim.run(n_trials=10)  # press Esc or close pygame window to quit
+    sim = Simulator(e, update_delay=0.5, display=True)  # create simulator (uses pygame when display=True, if available)
+    # NOTE: To speed up simulation, reduce update_delay and/or set display=False
+
+    sim.run(n_trials=100)  # run for a specified number of trials
+    # NOTE: To quit midway, press Esc or close pygame window, or hit Ctrl+C on the command-line
 
 
 if __name__ == '__main__':

projects/smartcab/smartcab/environment.py

@@ -29,6 +29,7 @@ class Environment(object):
     valid_actions = [None, 'forward', 'left', 'right']
     valid_inputs = {'light': TrafficLight.valid_states, 'oncoming': valid_actions, 'left': valid_actions, 'right': valid_actions}
     valid_headings = [(1, 0), (0, -1), (-1, 0), (0, 1)]  # ENWS
+    hard_time_limit = -100  # even if enforce_deadline is False, end trial when deadline reaches this value (to avoid deadlocks)
 
     def __init__(self):
         self.done = False
@@ -114,10 +115,14 @@ def step(self):
 
         self.t += 1
         if self.primary_agent is not None:
-            if self.enforce_deadline and self.agent_states[self.primary_agent]['deadline'] <= 0:
+            agent_deadline = self.agent_states[self.primary_agent]['deadline']
+            if agent_deadline <= self.hard_time_limit:
                 self.done = True
-                print "Environment.reset(): Primary agent could not reach destination within deadline!"
-            self.agent_states[self.primary_agent]['deadline'] -= 1
+                print "Environment.step(): Primary agent hit hard time limit ({})! Trial aborted.".format(self.hard_time_limit)
+            elif self.enforce_deadline and agent_deadline <= 0:
+                self.done = True
+                print "Environment.step(): Primary agent ran out of time! Trial aborted."
+            self.agent_states[self.primary_agent]['deadline'] = agent_deadline - 1
 
     def sense(self, agent):
         assert agent in self.agent_states, "Unknown agent!"
@@ -166,26 +171,33 @@ def act(self, agent, action):
         if action == 'forward':
             if light != 'green':
                 move_okay = False
-        elif action == 'left' and sense['oncoming']==None:
-            if light == 'green':
+        elif action == 'left':
+            if light == 'green' and (sense['oncoming'] == None or sense['oncoming'] == 'left'):
                 heading = (heading[1], -heading[0])
             else:
                 move_okay = False
-        elif action == 'right' and sense['left']==None:
-            heading = (-heading[1], heading[0])
+        elif action == 'right':
+            if light == 'green' or sense['left'] != 'straight':
+                heading = (-heading[1], heading[0])
+            else:
+                move_okay = False
 
-        if action is not None:
-            if move_okay:
+        if move_okay:
+            # Valid move (could be null)
+            if action is not None:
+                # Valid non-null move
                 location = ((location[0] + heading[0] - self.bounds[0]) % (self.bounds[2] - self.bounds[0] + 1) + self.bounds[0],
                             (location[1] + heading[1] - self.bounds[1]) % (self.bounds[3] - self.bounds[1] + 1) + self.bounds[1])  # wrap-around
                 #if self.bounds[0] <= location[0] <= self.bounds[2] and self.bounds[1] <= location[1] <= self.bounds[3]:  # bounded
                 state['location'] = location
                 state['heading'] = heading
-                reward = 2 if action == agent.get_next_waypoint() else 0.5
+                reward = 2.0 if action == agent.get_next_waypoint() else -0.5  # valid, but is it correct? (as per waypoint)
             else:
-                reward = -1
+                # Valid null move
+                reward = 0.0
         else:
-            reward = 1
+            # Invalid move
+            reward = -1.0
 
         if agent is self.primary_agent:
             if state['location'] == state['destination']:

projects/smartcab/smartcab/simulator.py

@@ -1,10 +1,13 @@
 import os
 import time
 import random
-import pygame
+import importlib
 
 class Simulator(object):
-    """PyGame-based simulator to create a dynamic environment."""
+    """Simulates agents in a dynamic smartcab environment.
+
+    Uses PyGame to display GUI, if available.
+    """
 
     colors = {
         'black'   : (  0,   0,   0),
@@ -18,13 +21,11 @@ class Simulator(object):
         'orange'  : (255, 128,   0)
     }
 
-    def __init__(self, env, size=None, frame_delay=10, update_delay=1.0, display=True):
+    def __init__(self, env, size=None, update_delay=1.0, display=True):
         self.env = env
         self.size = size if size is not None else ((self.env.grid_size[0] + 1) * self.env.block_size, (self.env.grid_size[1] + 1) * self.env.block_size)
         self.width, self.height = self.size
-        self.frame_delay = frame_delay
-        self.display = display
-
+
         self.bg_color = self.colors['white']
         self.road_width = 5
         self.road_color = self.colors['black']
@@ -35,17 +36,28 @@ def __init__(self, env, size=None, frame_delay=10, update_delay=1.0, display=Tru
         self.last_updated = 0.0
         self.update_delay = update_delay
 
-        pygame.init()
-        self.screen = pygame.display.set_mode(self.size)
-
-        self.agent_sprite_size = (32, 32)
-        self.agent_circle_radius = 10  # radius of circle, when using simple representation
-        for agent in self.env.agent_states:
-            agent._sprite = pygame.transform.smoothscale(pygame.image.load(os.path.join("images", "car-{}.png".format(agent.color))), self.agent_sprite_size)
-            agent._sprite_size = (agent._sprite.get_width(), agent._sprite.get_height())
-
-        self.font = pygame.font.Font(None, 28)
-        self.paused = False
+        self.display = display
+        if self.display:
+            try:
+                self.pygame = importlib.import_module('pygame')
+                self.pygame.init()
+                self.screen = self.pygame.display.set_mode(self.size)
+
+                self.frame_delay = max(1, int(self.update_delay * 1000))  # delay between GUI frames in ms (min: 1)
+                self.agent_sprite_size = (32, 32)
+                self.agent_circle_radius = 10  # radius of circle, when using simple representation
+                for agent in self.env.agent_states:
+                    agent._sprite = self.pygame.transform.smoothscale(self.pygame.image.load(os.path.join("images", "car-{}.png".format(agent.color))), self.agent_sprite_size)
+                    agent._sprite_size = (agent._sprite.get_width(), agent._sprite.get_height())
+
+                self.font = self.pygame.font.Font(None, 28)
+                self.paused = False
+            except ImportError as e:
+                self.display = False
+                print "Simulator.__init__(): Unable to import pygame; display disabled.\n{}: {}".format(e.__class__.__name__, e)
+            except Exception as e:
+                self.display = False
+                print "Simulator.__init__(): Error initializing GUI objects; display disabled.\n{}: {}".format(e.__class__.__name__, e)
 
     def run(self, n_trials=1):
         self.quit = False
@@ -56,31 +68,34 @@ def run(self, n_trials=1):
             self.last_updated = 0.0
             self.start_time = time.time()
             while True:
-                self.current_time = time.time() - self.start_time
-                #print "Simulator.run(): current_time = {:.3f}".format(self.current_time)
                 try:
-                    # Handle events
-                    for event in pygame.event.get():
-                        if event.type == pygame.QUIT:
-                            self.quit = True
-                        elif event.type == pygame.KEYDOWN:
-                            if event.key == 27:  # Esc
+                    # Update current time
+                    self.current_time = time.time() - self.start_time
+                    #print "Simulator.run(): current_time = {:.3f}".format(self.current_time)
+
+                    # Handle GUI events
+                    if self.display:
+                        for event in self.pygame.event.get():
+                            if event.type == self.pygame.QUIT:
                                 self.quit = True
-                            elif event.unicode == u' ':
-                                self.paused = True
+                            elif event.type == self.pygame.KEYDOWN:
+                                if event.key == 27:  # Esc
+                                    self.quit = True
+                                elif event.unicode == u' ':
+                                    self.paused = True
 
-                    if self.paused:
-                        self.pause()
+                        if self.paused:
+                            self.pause()
 
                     # Update environment
                     if self.current_time - self.last_updated >= self.update_delay:
                         self.env.step()
                         self.last_updated = self.current_time
 
-                    # Render and sleep
+                    # Render GUI and sleep
                     if self.display:
                         self.render()
-                        pygame.time.wait(self.frame_delay)
+                        self.pygame.time.wait(self.frame_delay)
                 except KeyboardInterrupt:
                     self.quit = True
                 finally:
@@ -97,16 +112,16 @@ def render(self):
         # Draw elements
         # * Static elements
         for road in self.env.roads:
-            pygame.draw.line(self.screen, self.road_color, (road[0][0] * self.env.block_size, road[0][1] * self.env.block_size), (road[1][0] * self.env.block_size, road[1][1] * self.env.block_size), self.road_width)
+            self.pygame.draw.line(self.screen, self.road_color, (road[0][0] * self.env.block_size, road[0][1] * self.env.block_size), (road[1][0] * self.env.block_size, road[1][1] * self.env.block_size), self.road_width)
 
         for intersection, traffic_light in self.env.intersections.iteritems():
-            pygame.draw.circle(self.screen, self.road_color, (intersection[0] * self.env.block_size, intersection[1] * self.env.block_size), 10)
+            self.pygame.draw.circle(self.screen, self.road_color, (intersection[0] * self.env.block_size, intersection[1] * self.env.block_size), 10)
             if traffic_light.state:  # North-South is open
-                pygame.draw.line(self.screen, self.colors['green'],
+                self.pygame.draw.line(self.screen, self.colors['green'],
                     (intersection[0] * self.env.block_size, intersection[1] * self.env.block_size - 15),
                     (intersection[0] * self.env.block_size, intersection[1] * self.env.block_size + 15), self.road_width)
             else:  # East-West is open
-                pygame.draw.line(self.screen, self.colors['green'],
+                self.pygame.draw.line(self.screen, self.colors['green'],
                     (intersection[0] * self.env.block_size - 15, intersection[1] * self.env.block_size),
                     (intersection[0] * self.env.block_size + 15, intersection[1] * self.env.block_size), self.road_width)
 
@@ -118,19 +133,19 @@ def render(self):
             agent_color = self.colors[agent.color]
             if hasattr(agent, '_sprite') and agent._sprite is not None:
                 # Draw agent sprite (image), properly rotated
-                rotated_sprite = agent._sprite if state['heading'] == (1, 0) else pygame.transform.rotate(agent._sprite, 180 if state['heading'][0] == -1 else state['heading'][1] * -90)
+                rotated_sprite = agent._sprite if state['heading'] == (1, 0) else self.pygame.transform.rotate(agent._sprite, 180 if state['heading'][0] == -1 else state['heading'][1] * -90)
                 self.screen.blit(rotated_sprite,
-                    pygame.rect.Rect(agent_pos[0] - agent._sprite_size[0] / 2, agent_pos[1] - agent._sprite_size[1] / 2,
+                    self.pygame.rect.Rect(agent_pos[0] - agent._sprite_size[0] / 2, agent_pos[1] - agent._sprite_size[1] / 2,
                         agent._sprite_size[0], agent._sprite_size[1]))
             else:
                 # Draw simple agent (circle with a short line segment poking out to indicate heading)
-                pygame.draw.circle(self.screen, agent_color, agent_pos, self.agent_circle_radius)
-                pygame.draw.line(self.screen, agent_color, agent_pos, state['location'], self.road_width)
+                self.pygame.draw.circle(self.screen, agent_color, agent_pos, self.agent_circle_radius)
+                self.pygame.draw.line(self.screen, agent_color, agent_pos, state['location'], self.road_width)
             if agent.get_next_waypoint() is not None:
                 self.screen.blit(self.font.render(agent.get_next_waypoint(), True, agent_color, self.bg_color), (agent_pos[0] + 10, agent_pos[1] + 10))
             if state['destination'] is not None:
-                pygame.draw.circle(self.screen, agent_color, (state['destination'][0] * self.env.block_size, state['destination'][1] * self.env.block_size), 6)
-                pygame.draw.circle(self.screen, agent_color, (state['destination'][0] * self.env.block_size, state['destination'][1] * self.env.block_size), 15, 2)
+                self.pygame.draw.circle(self.screen, agent_color, (state['destination'][0] * self.env.block_size, state['destination'][1] * self.env.block_size), 6)
+                self.pygame.draw.circle(self.screen, agent_color, (state['destination'][0] * self.env.block_size, state['destination'][1] * self.env.block_size), 15, 2)
 
         # * Overlays
         text_y = 10
@@ -139,18 +154,18 @@ def render(self):
             text_y += 20
 
         # Flip buffers
-        pygame.display.flip()
+        self.pygame.display.flip()
 
     def pause(self):
         abs_pause_time = time.time()
         pause_text = "[PAUSED] Press any key to continue..."
         self.screen.blit(self.font.render(pause_text, True, self.colors['cyan'], self.bg_color), (100, self.height - 40))
-        pygame.display.flip()
+        self.pygame.display.flip()
         print pause_text  # [debug]
         while self.paused:
-            for event in pygame.event.get():
-                if event.type == pygame.KEYDOWN:
+            for event in self.pygame.event.get():
+                if event.type == self.pygame.KEYDOWN:
                     self.paused = False
-            pygame.time.wait(self.frame_delay)
+            self.pygame.time.wait(self.frame_delay)
         self.screen.blit(self.font.render(pause_text, True, self.bg_color, self.bg_color), (100, self.height - 40))
         self.start_time += (time.time() - abs_pause_time)