Skip to content

Fix MDP class and add POMDP subclass and notebook #781

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
norvig merged 14 commits into from
Mar 2, 2018
Merged

Fix MDP class and add POMDP subclass and notebook #781

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
bd23fb7
Fixed typos and added inline LaTeX
bakerwho Feb 26, 2018
9d5ec3c
Fixed backslash for inline LaTeX
bakerwho Feb 27, 2018
d8af22b
Fixed more backslashes
bakerwho Feb 27, 2018
a09490d
Merge branch 'master' of https://github.com/aimacode/aima-python into…
bakerwho Feb 27, 2018
548c1cb
generalised MDP class and created POMDP notebook
bakerwho Feb 27, 2018
d04e14d
Fixed consistency issues with base MDP class
bakerwho Feb 28, 2018
bee037e
Small fix on CustomMDP
bakerwho Feb 28, 2018
97e51cd
Set default args to pass tests
bakerwho Feb 28, 2018
7e763e6
Added TableDrivenAgentProgram tests (#777)
ayushjain19 Feb 28, 2018
3fed661
Fixing tests
bakerwho Feb 28, 2018
bab969e
Fixed conflicts
bakerwho Feb 28, 2018
d778ec0
fixed conflicts
bakerwho Feb 28, 2018
35be708
fixed test_rl
bakerwho Feb 28, 2018
1a1a049
removed redundant code, fixed a comment
bakerwho Feb 28, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Set default args to pass tests
  • Loading branch information
@bakerwho
bakerwho committed Feb 28, 2018
commit 97e51cd1ccd588df74854b982feab3ef05526780
87 changes: 23 additions & 64 deletions mdp.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"from mdp import *\n",
Expand Down Expand Up @@ -67,9 +65,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"psource(MDP)"
Expand Down Expand Up @@ -142,13 +138,13 @@
"outputs": [],
"source": [
"class CustomMDP(MDP):\n",
" def __init__(self, init, terminals, reward, transition_matrix, gamma=.9):\n",
" def __init__(self, init, terminals, transition_matrix, reward = None, gamma=.9):\n",
" # All possible actions.\n",
" actlist = []\n",
" for state in transition_matrix.keys():\n",
" actlist.extend(transition_matrix[state])\n",
" actlist = list(set(actlist))\n",
" MDP.__init__(self, init, actlist, terminals, reward, transition_matrix, gamma=gamma)\n",
" MDP.__init__(self, init, actlist, terminals, transition_matrix, reward, gamma=gamma)\n",
"\n",
" def T(self, state, action):\n",
" if action is None:\n",
Expand All @@ -167,12 +163,10 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"our_mdp = CustomMDP(init, terminals, rewards, t, gamma=.9)"
"our_mdp = CustomMDP(init, terminals, t, rewards, gamma=.9)"
]
},
{
Expand All @@ -194,9 +188,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"psource(GridMDP)"
Expand Down Expand Up @@ -234,9 +226,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"sequential_decision_environment"
Expand All @@ -262,9 +252,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"psource(value_iteration)"
Expand Down Expand Up @@ -303,9 +291,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"value_iteration(sequential_decision_environment)"
Expand All @@ -321,9 +307,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"pseudocode(\"Value-Iteration\")"
Expand Down Expand Up @@ -421,7 +405,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true,
"scrolled": true
},
"outputs": [],
Expand Down Expand Up @@ -479,9 +462,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"psource(expected_utility)"
Expand All @@ -490,9 +471,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"psource(policy_iteration)"
Expand All @@ -514,9 +493,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"psource(policy_evaluation)"
Expand All @@ -532,9 +509,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"policy_iteration(sequential_decision_environment)"
Expand All @@ -543,9 +518,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"pseudocode('Policy-Iteration')"
Expand Down Expand Up @@ -612,9 +585,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"psource(GridMDP.T)"
Expand Down Expand Up @@ -652,9 +623,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"psource(GridMDP.to_arrows)"
Expand All @@ -671,9 +640,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"psource(GridMDP.to_grid)"
Expand Down Expand Up @@ -741,9 +708,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"from utils import print_table\n",
Expand Down Expand Up @@ -788,9 +753,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"pi = best_policy(sequential_decision_environment, value_iteration(sequential_decision_environment, .001))\n",
Expand Down Expand Up @@ -840,9 +803,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"pi = best_policy(sequential_decision_environment, value_iteration(sequential_decision_environment, .001))\n",
Expand Down Expand Up @@ -891,9 +852,7 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"metadata": {},
"outputs": [],
"source": [
"pi = best_policy(sequential_decision_environment, value_iteration(sequential_decision_environment, .001))\n",
Expand Down
14 changes: 9 additions & 5 deletions mdp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class MDP:
list of (p, s') pairs. We also keep track of the possible states,
terminal states, and actions for each state. [page 646]"""

def __init__(self, init, actlist, terminals, reward, transitions, states=None, gamma=.9):
def __init__(self, init, actlist, terminals, transitions = {}, reward = None, states=None, gamma=.9):

@norvig norvig Mar 2, 2018

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No spaces around = for keyword arguments.

if not (0 < gamma <= 1):
raise ValueError("An MDP must have 0 < gamma <= 1")

Expand All @@ -43,9 +43,13 @@ def __init__(self, init, actlist, terminals, reward, transitions, states=None, g

self.terminals = terminals
self.transitions = transitions
if self.transitions == {}:
print("Warning: Transition table is empty.")
self.gamma = gamma
self.reward = reward

if reward:
self.reward = reward
else:
self.reward = {s : 0 for s in self.states}
self.check_consistency()

def R(self, state):
Expand Down Expand Up @@ -122,8 +126,8 @@ def __init__(self, grid, terminals, init=(0, 0), gamma=.9):
for a in actlist:
transitions[s][a] = self.calculate_T(s, a)
MDP.__init__(self, init, actlist=actlist,
terminals=terminals, reward = reward,
transitions = transitions, states = states, gamma=gamma)
terminals=terminals, transitions = transitions,
reward = reward, states = states, gamma=gamma)

def calculate_T(self, state, action):
if action is None:
Expand Down