Modeling Agents with Probabilistic Programs

Owain Evans
Andreas Stuhlmüller
John Salvatier
Daniel Filan

meetup-2017.md

---
layout: chapter
title: Modeling Agents & Reinforcement Learning with Probabilistic Programming
hidden: true
---

## Intro

### Motivation

Why probabilistic programming?
- **ML:** predictions based on prior assumptions and data
- **Deep Learning:** lots of data + very weak assumptions
- **Rule-based systems:** strong assumptions + little data
- **Probabilistic programming:** a flexible middle ground

Why model agents?
- Build **artificial agents** to automate decision-making
    - Example: stock trading
- **Model humans** to build helpful ML systems
    - Examples: recommendation systems, dialog systems

### Preview

What to get out of this talk:
- Intuition for programming in a PPL
- Core PPL concepts
- Why are PPLs uniquely suited for modeling agents?
- Idioms for writing agents as PPs
- How do RL and PP relate?

What not to expect:
- Lots of applications
- Production-ready systems

## Probabilistic programming basics

### Our language: WebPPL

Try it at [webppl.org](http://webppl.org)

### A functional subset of JavaScript

Why JS?
- Fast
- Rich ecosystem
- Actually a nice language underneath all the cruft
- Runs locally via node.js, but also in browser:
    - [SmartPages](https://stuhlmueller.org/smartpages/)
    - [Image inference viz](http://dippl.org/examples/vision.html)
    - [Spaceships](http://dritchie.github.io/web-procmod/)
    - [Agent viz](http://agentmodels.org/chapters/3b-mdp-gridworld.html#hiking-in-gridworld)

~~~~
var xs = [1, 2, 3, 4];

var square = function(x) {
  return x * x;
};

map(square, xs);
~~~~

### Distributions and sampling

Docs: [distributions](http://docs.webppl.org/en/dev/distributions.html)

#### Discrete distributions

Examples: `Bernoulli`, `Categorical`

Sampling helpers: `flip`, `categorical`

~~~~
var dist = Bernoulli({ p: 0.3 });

var flip = function(p) {
  return sample(Bernoulli({ p }));
}

flip(.3)
~~~~

#### Continuous distributions

Examples: `Gaussian`, `Beta`

~~~~
var dist = Gaussian({
  mu: 1,
  sigma: 0.5
});

viz(repeat(1000, function() { return sample(dist); }));
~~~~

#### Building complex distributions out of simple parts

Example: geometric distribution

~~~~
var geometric = function(p) {
  if (flip(p)) {
    return 0;
  } else {
    return 1 + geometric(p);
  }
};

viz(repeat(100, function() { return geometric(.5); }));
~~~~

### Inference

#### Reifying distributions

`Infer` reifies the geometric distribution so that we can compute probabilities:

~~~~
var geometric = function(p) {
  if (flip(p)) {
    return 0;
  } else {
    return 1 + geometric(p);
  }
};

var model = function() {
  return geometric(.5);
};

var dist = Infer({
  model,
  maxExecutions: 100
});

viz(dist);

Math.exp(dist.score(3))
~~~~

#### Computing conditional distributions

Example: inferring the weight of a geometric distribution

~~~~
var geometric = function(p) {
  if (flip(p)) {
    return 0;
  } else {
    return 1 + geometric(p);
  }
}

var model = function() {
  var u = uniform(0, 1);
  var x = geometric(u);
  condition(x < 4);
  return u;
}

var dist = Infer({
  model,
  method: 'rejection',
  samples: 1000
})

dist
~~~~

#### Technical note: three ways to condition

~~~~
var model = function() {
  var p = flip(.5) ? 0.5 : 1;
  var coin = Bernoulli({ p });

  var x = sample(coin);
  condition(x === true);

//  observe(coin, true);

//  factor(coin.score(true));

  return { p };
}

viz.table(Infer({ model }));
~~~~

#### A slightly less toy example: regression

Docs: [inference algorithms](http://docs.webppl.org/en/master/inference/methods.html)

~~~~
var xs = [1, 2, 3, 4, 5];
var ys = [2, 4, 6, 8, 10];

var model = function() {
  var slope = gaussian(0, 10);
  var offset = gaussian(0, 10);
  var f = function(x) {
    var y = slope * x + offset;
    return Gaussian({ mu: y, sigma: .1 })
  };
  map2(function(x, y){
    observe(f(x), y)
  }, xs, ys)
  return { slope, offset };
}

Infer({
  model,
  method: 'MCMC',
  kernel: {HMC: {steps: 10, stepSize: .01}},
  samples: 2000,
})
~~~~

## Agents as probabilistic programs

### Deterministic choices

~~~~
var actions = ['italian', 'french'];

var outcome = function(action) {
  if (action === 'italian') {
    return 'pizza';
  } else {
    return 'steak frites';
  }
};

var actionDist = Infer({
  model() {
    var action = uniformDraw(actions);
    condition(outcome(action) === 'pizza');
    return action;
  }
});

actionDist
~~~~

### Expected utility

~~~~
var actions = ['italian', 'french'];

var transition = function(state, action) {
  var nextStates = ['bad', 'good', 'spectacular'];
  var nextProbs = ((action === 'italian') ?
                   [0.2, 0.6, 0.2] :
                   [0.05, 0.9, 0.05]);
  return categorical(nextProbs, nextStates);
};

var utility = function(state) {
  var table = {
    bad: -10,
    good: 6,
    spectacular: 8
  };
  return table[state];
};

var expectedUtility = function(action) {
  var utilityDist = Infer({
    model: function() {
      var nextState = transition('initialState', action);
      var u = utility(nextState);
      return u;
    }
  });
  return expectation(utilityDist);
};

map(expectedUtility, actions);
~~~~

### Softmax-optimal decision-making

~~~~
var actions = ['italian', 'french'];

var transition = function(state, action) {
  var nextStates = ['bad', 'good', 'spectacular'];
  var nextProbs = ((action === 'italian') ?
                   [0.2, 0.6, 0.2] :
                   [0.05, 0.9, 0.05]);
  return categorical(nextProbs, nextStates);
};

var utility = function(state) {
  var table = {
    bad: -10,
    good: 6,
    spectacular: 8
  };
  return table[state];
};

var alpha = 1;

var agent = function(state) {
  return Infer({
    model() {

      var action = uniformDraw(actions);

      var expectedUtility = function(action) {
        var utilityDist = Infer({
          model: function() {
            var nextState = transition('initialState', action);
            var u = utility(nextState);
            return u;
          }
        });
        return expectation(utilityDist);
      };

      var eu = expectedUtility(action);

      factor(eu);

      return action;

    }
  });
};

agent('initialState');
~~~~

## Sequential decision problems

- [Restaurant Gridworld](http://agentmodels.org/chapters/3a-mdp.html) (1, last)
- Structure of expected utility recursion
- Dynamic programming

~~~~
var act = function(state) {
  return Infer({ model() {
    var action = uniformDraw(stateToActions(state));
    var eu = expectedUtility(state, action);
    factor(eu);
    return action;
  }});
};

var expectedUtility = function(state, action){
  var u = utility(state, action);
  if (isTerminal(state)){
    return u;
  } else {
    return u + expectation(Infer({ model() {
      var nextState = transition(state, action);
      var nextAction = sample(act(nextState));
      return expectedUtility(nextState, nextAction);
    }}));
  }
};
~~~~

- [Hiking Gridworld](http://agentmodels.org/chapters/3b-mdp-gridworld.html) (1, 2, 3, last)
- Expected state-action utilities (Q values)
- [Temporal inconsistency](http://agentmodels.org/chapters/5b-time-inconsistency.html) in Restaurant Gridworld

## Reasoning about agents

- [Learning about preferences from observations](http://agentmodels.org/chapters/4-reasoning-about-agents.html) (1 & 2)

## Multi-agent models

### A simple example: Coordination games

~~~~
var locationPrior = function() {
  if (flip(.55)) {
    return 'popular-bar';
  } else {
    return 'unpopular-bar';
  }
}

var alice = dp.cache(function(depth) {
  return Infer({ model() {
    var myLocation = locationPrior();
    var bobLocation = sample(bob(depth - 1));
    condition(myLocation === bobLocation);
    return myLocation;
  }});
});

var bob = dp.cache(function(depth) {
  return Infer({ model() {
    var myLocation = locationPrior();
    if (depth === 0) {
      return myLocation;
    } else {
      var aliceLocation = sample(alice(depth));
      condition(myLocation === aliceLocation);
      return myLocation;
    }
  }});
});

alice(5)
~~~~

### Other examples

- [Game playing: tic-tac-toe](http://agentmodels.org/chapters/7-multi-agent.html)
- [Language understanding](http://agentmodels.org/chapters/7-multi-agent.html)

## Reinforcement learning

### Algorithms vs Models

- Models: encode world knowledge
    - PPLs suited for expressing models
- Algorithms: encode mechanisms (for inference, optimization)
    - RL is mostly about algorithms
- But some algorithms can be expressed using PPL components

### Inference vs. Optimization

~~~~
var k = 3;  // number of heads
var n = 10; // number of coin flips

var model = function() {
  var p = sample(Uniform({ a: 0, b: 1}));
  var dist = Binomial({ p, n });
  observe(dist, k);
  return p;
};

var dist = Infer({
  model,
  method: ‘MCMC',
  samples: 100000,
  burn: 1000
});

expectation(dist);
~~~~

~~~~
var k = 3;  // number of heads
var n = 10; // number of coin flips

var model = function() {
  var p = Math.sigmoid(modelParam({ name: 'p' }));
  var dist = Binomial({ p, n });
  observe(dist, k);
  return p;
};

Optimize({
  model,
  steps: 1000,
  optMethod: { sgd: { stepSize: 0.01 }}
});

Math.sigmoid(getParams().p);
~~~~

### Policy Gradient

~~~~
///fold:
var numArms = 10;

var meanRewards = map(
  function(i) {
    if ((i === 7) || (i === 3)) {
      return 5;
    } else {
      return 0;
    }
  },
  _.range(numArms));

var blackBox = function(action) {
  var mu = meanRewards[action];
  var u = Gaussian({ mu, sigma: 0.01 }).sample();
  return u;
};
///

// actions: [0, 1, 2, ..., 9]

// blackBox: action -> utility

var agent = function() {
  var ps = softmax(modelParam({ dims: [numArms, 1], name: 'ps' }));
  var action = sample(Discrete({ ps }));
  var utility = blackBox(action);
  factor(utility);
  return action;
};

Optimize({ model: agent, steps: 10000 });

var params = getParams();
viz.bar(
  _.range(10),
  _.flatten(softmax(params.ps[0]).toArray()));
~~~~

## Conclusion

What to get out of this talk, revisited:

- **Intuition for programming in a PPL**
- **Core PPL concepts**
    - Distributions & samplers
    - Inference turns samplers into distributions
    - `sample` turns distributions into samples
    - Optimization fits free parameters
- **Idioms for writing agents as probabilistic programs**
    - Planning as inference
    - Sequential planning via recursion into the future
    - Multi-agent planning via recursion into other agents' minds
- **Why are PPLs uniquely suited for modeling agents?**
    - Agents are structured programs
    - Planning via nested conditional distributions
- **How do RL and PP relate?**
    - Algorithms vs models
    - Policy gradient as a PP

Where to go from here:
- [WebPPL](http://webppl.org) (webppl.org)
- [AgentModels](http://agentmodels.org) (agentmodels.org)
- andreas@ought.com


5e-joint-inference.md

---
layout: chapter
title: Joint inference of biases and preferences II
description: Explaining temptation and pre-commitment using softmax noise and hyperbolic discounting.

---

## Restaurant Choice: Time-inconsistent vs. optimal MDP agents

Returning to the MDP Restaurant Choice problem, we compare a model that assumes an optimal, non-discounting MDP agent to a model that includes both time-inconsistent and optimal agents. We also consider models that expand the set of preferences the agent can have.

### Assume discounting, infer "Naive" or "Sophisticated"

Before making a direct comparison, we demonstrate that we can infer the preferences of time-inconsistent agents from observations of their behavior.

First we condition on the path where the agent moves to Donut North. We call this the Naive path because it is distinctive to the Naive hyperbolic discounter (who is tempted by Donut North on the way to Veg):

~~~~
///fold: restaurant choice MDP, naiveTrajectory
var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});

var naiveTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"u"],
  [{"loc":[3,4],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"u"],
  [{"loc":[3,5],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[3,4]},"l"],
  [{"loc":[2,5],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[3,5],"timeAtRestaurant":0},"l"],
  [{"loc":[2,5],"terminateAfterAction":true,"timeLeft":6,"previousLoc":[2,5],"timeAtRestaurant":1},"l"]
];
///
viz.gridworld(mdp.world, { trajectory: naiveTrajectory });
~~~~

For inference, we specialize the approach in the previous chapter for agents in MDPs that are potentially time inconsistent. So we infer $$\nu$$ and $$k$$ (the hyperbolic discounting parameters) but not the initial belief state $$b_0$$. The function `exampleGetPosterior` is a slightly simplified version of the library function we use below.

~~~~
var exampleGetPosterior = function(mdp, prior, observedStateAction){
  var world = mdp.world;
  var makeUtilityFunction = mdp.makeUtilityFunction;
  return Infer({ model() {

    // Sample parameters from prior
    var priorUtility = prior.priorUtility;
    var utilityTable = priorUtility();
    var priorDiscounting = prior.discounting
    var sophisticatedOrNaive = priorDiscounting().sophisticatedOrNaive;

    var priorAlpha = prior.priorAlpha;

    // Create agent with those parameters
    var agent = makeMDPAgent({
      utility: makeUtilityFunction(utilityTable),
      alpha: priorAlpha(),
      discount: priorDiscounting().discount,
      sophisticatedOrNaive : sophisticatedOrNaive
    }, world);

    var agentAction = agent.act;

    // Condition on observed actions
    map(function(stateAction) {
      var state = stateAction[0];
      var action = stateAction[1];
      observe(agentAction(state, 0), action);
    }, observedStateAction);

    // return parameters and summary statistics
    var vegMinusDonut = sum(utilityTable['Veg']) - sum(utilityTable['Donut N']);

    return {
      utility: utilityTable,
      sophisticatedOrNaive: discounting.sophisticatedOrNaive,
      discount: discounting.discount,
      alpha,
      vegMinusDonut,
    };
  }});
};
~~~~

This inference function allows for inference over the softmax parameter ($$\alpha$$ or `alpha`) and the discount constant ($$k$$ or `discount`). For this example, we fix these values so that the agent has low noise ($$\alpha=1000$$) and so $$k=1$$. We also fix the `timeCost` utility to be small and negative and Noodle's utility to be negative. We infer only the agent's utilities and whether they are Naive or Sophisticated.

~~~~
///fold: Call to hyperbolic library function and helper display function
var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});

var naiveTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"u"],
  [{"loc":[3,4],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"u"],
  [{"loc":[3,5],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[3,4]},"l"],
  [{"loc":[2,5],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[3,5],"timeAtRestaurant":0},"l"],
  [{"loc":[2,5],"terminateAfterAction":true,"timeLeft":6,"previousLoc":[2,5],"timeAtRestaurant":1},"l"]
];

var restaurantHyperbolicInfer = getRestaurantHyperbolicInfer();
var getPosterior = restaurantHyperbolicInfer.getPosterior;

var displayResults = function(priorDist, posteriorDist) {

  var priorUtility = priorDist.MAP().val.utility;
  print('Prior highest-probability utility for Veg: ' + priorUtility['Veg']
        + '. Donut: ' + priorUtility['Donut N'] + ' \n');

  var posteriorUtility = posteriorDist.MAP().val.utility;
  print('Posterior highest-probability utility for Veg: '
        + posteriorUtility['Veg'] + '. Donut: ' + posteriorUtility['Donut N']
        + ' \n');

  var getPriorProb = function(x){
    var label = _.keys(x)[0];
    var dist = getMarginalObject(priorDist, label);
    return Math.exp(dist.score(x));
  };

  var getPosteriorProb = function(x){
    var label = _.keys(x)[0];
    var dist = getMarginalObject(posteriorDist, label);
    return Math.exp(dist.score(x));
  };

  var sophisticationPriorDataTable = map(
    function(x) {
      return {
        sophisticatedOrNaive: x,
        probability: getPriorProb({sophisticatedOrNaive: x}),
        distribution: 'prior'
      };
    },
    ['naive', 'sophisticated']);

  var sophisticationPosteriorDataTable = map(
    function(x){
      return {
        sophisticatedOrNaive: x,
        probability: getPosteriorProb({sophisticatedOrNaive: x}),
        distribution: 'posterior'
      };
    },
    ['naive', 'sophisticated']);

  var sophisticatedOrNaiveDataTable = append(sophisticationPosteriorDataTable,
                                             sophisticationPriorDataTable);

  viz.bar(sophisticatedOrNaiveDataTable, { groupBy: 'distribution' });

  var vegMinusDonutPriorDataTable = map(
    function(x){
      return {
        vegMinusDonut: x,
        probability: getPriorProb({vegMinusDonut: x}),
        distribution: 'prior'
      };
    },
    [-60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60]);

  var vegMinusDonutPosteriorDataTable = map(
    function(x){
      return {
        vegMinusDonut: x,
        probability: getPosteriorProb({vegMinusDonut: x}),
        distribution: 'posterior'
      };
    },
    [-60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60]);

  var vegMinusDonutDataTable = append(vegMinusDonutPriorDataTable,
                                      vegMinusDonutPosteriorDataTable);

  viz.bar(vegMinusDonutDataTable, { groupBy: 'distribution' });

  var donutTemptingPriorDataTable = map(
    function(x) {
      return {
        donutTempting: x,
        probability: getPriorProb({donutTempting: x}),
        distribution: 'prior'
      };
    },
    [true, false]);

  var donutTemptingPosteriorDataTable = map(
    function(x) {
      return {
        donutTempting: x,
        probability: getPosteriorProb({donutTempting: x}),
        distribution: 'posterior'
      };
    },
    [true, false]);

  var donutTemptingDataTable = append(donutTemptingPriorDataTable,
                                      donutTemptingPosteriorDataTable);

  viz.bar(donutTemptingDataTable, { groupBy: 'distribution' });
};
///

// Prior on agent's utility function: each restaurant has an
// *immediate* utility and a *delayed* utility (which is received after a
// delay of 1).
var priorUtility = function(){
  var utilityValues =  [-10, 0, 10, 20];
  var donut = [uniformDraw(utilityValues), uniformDraw(utilityValues)];
  var veg = [uniformDraw(utilityValues), uniformDraw(utilityValues)];
  return {
    'Donut N': donut,
    'Donut S': donut,
    'Veg': veg,
    'Noodle': [-10, -10],
    'timeCost': -.01
  };
};

var priorDiscounting = function(){
  return {
    discount: 1,
    sophisticatedOrNaive: uniformDraw(['naive', 'sophisticated'])
  };
};
var priorAlpha = function(){ return 1000; };
var prior = {
  utility: priorUtility,
  discounting: priorDiscounting,
  alpha: priorAlpha
};

// Get world and observations
var posterior = getPosterior(mdp.world, prior, naiveTrajectory);

// To get the prior, we condition on the empty list of observations
displayResults(getPosterior(mdp.world, prior, []), posterior);
~~~~

We display maximum values and marginal distributions for both the prior and the posterior conditioned on the path shown above. To compute the prior, we simply condition on the empty list of observations.

The first graph shows the distribution over whether the agent is Sophisticated or Naive (labeled `sophisticatedOrNaive`). For the other graphs, we compute summary statistics of the agent's parameters and display the distribution over them. The variable `vegMinusDonut` is the difference in *total* utility between Veg and Donut, ignoring the fact that each restaurant has an *immediate* and *delayed* utility. Inference rules out cases where the total utility is equal (which is most likely in the prior), since the agent would simply go to Donut South in that case. Finally, we introduce a variable `donutTempting`, which is true if the agent prefers Veg to Donut North at the start but reverses this preference when adjacent to Donut North. The prior probability of `donutTempting` is less than $$0.1$$, since it depends on relatively delicate balance of utilities and the discounting behavior. The posterior is closer to $$0.9$$, suggesting (along with the posterior on `sophisticatedOrNaive`) that this is the explanation of the data favored by the model.

--------

Using the same prior, we condition on the "Sophisticated" path (i.e. the path distinctive to the Sophisticated agent who avoids the temptation of Donut North and takes the long route to Veg):

~~~~
///fold:
var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});

var sophisticatedTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"r"],
  [{"loc":[4,3],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"r"],
  [{"loc":[5,3],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[4,3]},"u"],
  [{"loc":[5,4],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[5,3]},"u"],
  [{"loc":[5,5],"terminateAfterAction":false,"timeLeft":5,"previousLoc":[5,4]},"u"],
  [{"loc":[5,6],"terminateAfterAction":false,"timeLeft":4,"previousLoc":[5,5]},"l"],
  [{"loc":[4,6],"terminateAfterAction":false,"timeLeft":3,"previousLoc":[5,6]},"u"],
  [{"loc":[4,7],"terminateAfterAction":false,"timeLeft":2,"previousLoc":[4,6],"timeAtRestaurant":0},"l"],
  [{"loc":[4,7],"terminateAfterAction":true,"timeLeft":2,"previousLoc":[4,7],"timeAtRestaurant":1},"l"]
];
///
viz.gridworld(mdp.world, { trajectory: sophisticatedTrajectory });
~~~~

Here are the results of inference:

~~~~
///fold: Definition of world, prior and inference function is same as above codebox
var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});

var restaurantHyperbolicInfer = getRestaurantHyperbolicInfer();
var getPosterior = restaurantHyperbolicInfer.getPosterior;

var displayResults = function(priorDist, posteriorDist) {

  var priorUtility = priorDist.MAP().val.utility;
  print('Prior highest-probability utility for Veg: ' + priorUtility['Veg']
        + '. Donut: ' + priorUtility['Donut N'] + ' \n');

  var posteriorUtility = posteriorDist.MAP().val.utility;
  print('Posterior highest-probability utility for Veg: '
        + posteriorUtility['Veg'] + '. Donut: ' + posteriorUtility['Donut N']
        + ' \n');

  var getPriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(priorDist, label);
    return Math.exp(dist.score(x));
  };

  var getPosteriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(posteriorDist, label);
    return Math.exp(dist.score(x));
  };

  var sophisticationPriorDataTable = map(
    function(x){
      return {
        sophisticatedOrNaive: x,
        probability: getPriorProb({sophisticatedOrNaive: x}),
        distribution: 'prior'
      };
    },
    ['naive', 'sophisticated']);

  var sophisticationPosteriorDataTable = map(
    function(x){
      return {
        sophisticatedOrNaive: x,
        probability: getPosteriorProb({sophisticatedOrNaive: x}),
        distribution: 'posterior'
      };
    },
    ['naive', 'sophisticated']);

  var sophisticatedOrNaiveDataTable = append(sophisticationPriorDataTable,
                                             sophisticationPosteriorDataTable);

  viz.bar(sophisticatedOrNaiveDataTable, { groupBy: 'distribution' });

  var vegMinusDonutPriorDataTable = map(
    function(x) {
      return {
        vegMinusDonut: x,
        probability: getPriorProb({vegMinusDonut: x}),
        distribution: 'prior'
      };
    },
    [-60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60]);

  var vegMinusDonutPosteriorDataTable = map(
    function(x) {
      return {
        vegMinusDonut: x,
        probability: getPosteriorProb({vegMinusDonut: x}),
        distribution: 'posterior'
      };
    },
    [-60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60]);

  var vegMinusDonutDataTable = append(vegMinusDonutPriorDataTable,
                                      vegMinusDonutPosteriorDataTable);

  viz.bar(vegMinusDonutDataTable, { groupBy: 'distribution' });

  var donutTemptingPriorDataTable = map(
    function(x) {
      return {
        donutTempting: x,
        probability: getPriorProb({ donutTempting: x }),
        distribution: 'prior'
      };
    },
    [true, false]);

  var donutTemptingPosteriorDataTable = map(
    function(x) {
      return {
        donutTempting: x,
        probability: getPosteriorProb({ donutTempting: x }),
        distribution: 'posterior'
      };
    },
    [true, false]);

  var donutTemptingDataTable = append(donutTemptingPriorDataTable,
                                      donutTemptingPosteriorDataTable);

  viz.bar(donutTemptingDataTable, { groupBy: 'distribution' });
};

// Prior on agent's utility function
var priorUtility = function() {
  var utilityValues =  [-10, 0, 10, 20];
  var donut = [uniformDraw(utilityValues), uniformDraw(utilityValues)];
  var veg = [uniformDraw(utilityValues), uniformDraw(utilityValues)];
  return {
    'Donut N': donut,
    'Donut S': donut,
    'Veg': veg,
    'Noodle': [-10, -10],
    'timeCost': -.01
  };
};

var priorDiscounting = function(){
  return {
    discount: 1,
    sophisticatedOrNaive: uniformDraw(['naive','sophisticated'])
  };
};
var priorAlpha = function(){ return 1000; };
var prior = {
  utility: priorUtility,
  discounting: priorDiscounting,
  alpha: priorAlpha
};

var sophisticatedTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"r"],
  [{"loc":[4,3],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"r"],
  [{"loc":[5,3],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[4,3]},"u"],
  [{"loc":[5,4],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[5,3]},"u"],
  [{"loc":[5,5],"terminateAfterAction":false,"timeLeft":5,"previousLoc":[5,4]},"u"],
  [{"loc":[5,6],"terminateAfterAction":false,"timeLeft":4,"previousLoc":[5,5]},"l"],
  [{"loc":[4,6],"terminateAfterAction":false,"timeLeft":3,"previousLoc":[5,6]},"u"],
  [{"loc":[4,7],"terminateAfterAction":false,"timeLeft":2,"previousLoc":[4,6],"timeAtRestaurant":0},"l"],
  [{"loc":[4,7],"terminateAfterAction":true,"timeLeft":2,"previousLoc":[4,7],"timeAtRestaurant":1},"l"]
];
///

// Get world and observations
var posterior = getPosterior(mdp.world, prior, sophisticatedTrajectory);
displayResults(getPosterior(mdp.world, prior, []), posterior);
~~~~

If the agent goes directly to Veg, then they don't provide information about whether they are Naive or Sophisticated. Using the same prior again, we do inference on this path:

~~~~
///fold:
var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});

var vegDirectTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"u"],
  [{"loc":[3,4],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"u"],
  [{"loc":[3,5],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[3,4]},"u"],
  [{"loc":[3,6],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[3,5]},"r"],
  [{"loc":[4,6],"terminateAfterAction":false,"timeLeft":5,"previousLoc":[3,6]},"u"],
  [{"loc":[4,7],"terminateAfterAction":false,"timeLeft":4,"previousLoc":[4,6],"timeAtRestaurant":0},"l"],
  [{"loc":[4,7],"terminateAfterAction":true,"timeLeft":4,"previousLoc":[4,7],"timeAtRestaurant":1},"l"]
];
///
viz.gridworld(mdp.world, { trajectory: vegDirectTrajectory });
~~~~

Here are the results of inference:

~~~~
// Definition of world, prior and inference function is same as above codebox

///fold:
var restaurantHyperbolicInfer = getRestaurantHyperbolicInfer();
var getPosterior = restaurantHyperbolicInfer.getPosterior;

var displayResults = function(priorDist, posteriorDist) {

  var priorUtility = priorDist.MAP().val.utility;
  print('Prior highest-probability utility for Veg: ' + priorUtility['Veg']
    + '. Donut: ' + priorUtility['Donut N'] + ' \n');

  var posteriorUtility = posteriorDist.MAP().val.utility;
  print('Posterior highest-probability utility for Veg: '
    + posteriorUtility['Veg'] + '. Donut: ' + posteriorUtility['Donut N']
    + ' \n');

  var getPriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(priorDist, label);
    return Math.exp(dist.score(x));
  };

  var getPosteriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(posteriorDist, label);
    return Math.exp(dist.score(x));
  };

  var sophisticationPriorDataTable = map(function(x) {
    return {sophisticatedOrNaive: x,
            probability: getPriorProb({sophisticatedOrNaive: x}),
            distribution: 'prior'};
  }, ['naive', 'sophisticated']);

  var sophisticationPosteriorDataTable = map(function(x) {
    return {sophisticatedOrNaive: x,
            probability: getPosteriorProb({sophisticatedOrNaive: x}),
            distribution: 'posterior'};
  }, ['naive', 'sophisticated']);

  var sophisticatedOrNaiveDataTable = append(sophisticationPriorDataTable,
                                             sophisticationPosteriorDataTable);

  viz.bar(sophisticatedOrNaiveDataTable, { groupBy: 'distribution' });

  var vegMinusDonutPriorDataTable = map(function(x){
    return {
      vegMinusDonut: x,
      probability: getPriorProb({vegMinusDonut: x}),
      distribution: 'prior'
    };
  }, [-60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60]);

  var vegMinusDonutPosteriorDataTable = map(function(x){
    return {vegMinusDonut: x,
            probability: getPosteriorProb({vegMinusDonut: x}),
            distribution: 'posterior'};
  }, [-60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60]);

  var vegMinusDonutDataTable = append(vegMinusDonutPriorDataTable,
                                      vegMinusDonutPosteriorDataTable);

  viz.bar(vegMinusDonutDataTable, {groupBy: 'distribution'});

  var donutTemptingPriorDataTable = map(function(x){
    return {
      donutTempting: x,
      probability: getPriorProb({donutTempting: x}),
      distribution: 'prior'
    };
  }, [true, false]);

  var donutTemptingPosteriorDataTable = map(function(x){
    return {
      donutTempting: x,
      probability: getPosteriorProb({donutTempting: x}),
      distribution: 'posterior'
    };
  }, [true, false]);

  var donutTemptingDataTable = append(donutTemptingPriorDataTable,
                                      donutTemptingPosteriorDataTable);

  viz.bar(donutTemptingDataTable, { groupBy: 'distribution' });
};

// Prior on agent's utility function
var priorUtility = function() {
  var utilityValues =  [-10, 0, 10, 20];
  var donut = [uniformDraw(utilityValues), uniformDraw(utilityValues)];
  var veg = [uniformDraw(utilityValues), uniformDraw(utilityValues)];
  return {
    'Donut N': donut,
    'Donut S': donut,
    'Veg': veg,
    'Noodle': [-10, -10],
    'timeCost': -.01
  };
};

var priorDiscounting = function() {
  return {
    discount: 1,
    sophisticatedOrNaive: uniformDraw(['naive','sophisticated'])
  };
};
var priorAlpha = function(){return 1000;};
var prior = {
  utility: priorUtility,
  discounting: priorDiscounting,
  alpha: priorAlpha
};

var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});

var vegDirectTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"u"],
  [{"loc":[3,4],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"u"],
  [{"loc":[3,5],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[3,4]},"u"],
  [{"loc":[3,6],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[3,5]},"r"],
  [{"loc":[4,6],"terminateAfterAction":false,"timeLeft":5,"previousLoc":[3,6]},"u"],
  [{"loc":[4,7],"terminateAfterAction":false,"timeLeft":4,"previousLoc":[4,6],"timeAtRestaurant":0},"l"],
  [{"loc":[4,7],"terminateAfterAction":true,"timeLeft":4,"previousLoc":[4,7],"timeAtRestaurant":1},"l"]
];
///

var posterior = getPosterior(mdp.world, prior, vegDirectTrajectory);
displayResults(getPosterior(mdp.world, prior, []), posterior);
~~~~

---------

### Assume non-discounting, infer preferences and softmax

We want to compare a model that assumes an optimal MDP agent with one that allows for time-inconsistency. We first show the inferences by the model that assumes optimality. This model can only explain the anomalous Naive and Sophisticated paths in terms of softmax noise (lower values for $$\alpha$$). We display the prior and posteriors for both the Naive and Sophisticated paths.

~~~~
///fold:
var restaurantHyperbolicInfer = getRestaurantHyperbolicInfer();
var getPosterior = restaurantHyperbolicInfer.getPosterior;

var displayResults = function(priorDist, posteriorDist) {

  var priorUtility = priorDist.MAP().val.utility;
  print('Prior highest-probability utility for Veg: ' + priorUtility['Veg']
        + '. Donut: ' + priorUtility['Donut N'] + ' \n');

  var posteriorUtility = posteriorDist.MAP().val.utility;
  print('Posterior highest-probability utility for Veg: '
        + posteriorUtility['Veg'] + '. Donut: ' + posteriorUtility['Donut N']
        + ' \n');

  var getPriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(priorDist, label);
    return Math.exp(dist.score(x));
  };

  var getPosteriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(posteriorDist, label);
    return Math.exp(dist.score(x));
  };

  var vegMinusDonutPriorDataTable = map(
    function(x){
      return {
        vegMinusDonut: x,
        probability: getPriorProb({ vegMinusDonut: x }),
        distribution: 'prior'
      };
    },
    [-50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50]);

  var vegMinusDonutPosteriorDataTable = map(
    function(x){
      return {
        vegMinusDonut: x,
        probability: getPosteriorProb({ vegMinusDonut: x }),
        distribution: 'posterior'
      };
    },
    [-50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50]);

  var vegMinusDonutDataTable = append(vegMinusDonutPriorDataTable,
                                      vegMinusDonutPosteriorDataTable);

  viz.bar(vegMinusDonutDataTable, { groupBy: 'distribution' });

  var alphaPriorDataTable = map(
    function(x){
      return {
        alpha: x,
        probability: getPriorProb({ alpha: x }),
        distribution: 'prior'
      };
    },
    [0.1, 10, 100, 1000]);

  var alphaPosteriorDataTable = map(
    function(x){
      return {
        alpha: x,
        probability: getPosteriorProb({ alpha: x }),
        distribution: 'posterior'
      };
    },
    [0.1, 10, 100, 1000]);

  var alphaDataTable = append(alphaPriorDataTable,
                              alphaPosteriorDataTable);

  viz.bar(alphaDataTable, { groupBy: 'distribution' });
};

var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});

var naiveTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"u"],
  [{"loc":[3,4],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"u"],
  [{"loc":[3,5],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[3,4]},"l"],
  [{"loc":[2,5],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[3,5],"timeAtRestaurant":0},"l"],
  [{"loc":[2,5],"terminateAfterAction":true,"timeLeft":6,"previousLoc":[2,5],"timeAtRestaurant":1},"l"]
];

var sophisticatedTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"r"],
  [{"loc":[4,3],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"r"],
  [{"loc":[5,3],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[4,3]},"u"],
  [{"loc":[5,4],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[5,3]},"u"],
  [{"loc":[5,5],"terminateAfterAction":false,"timeLeft":5,"previousLoc":[5,4]},"u"],
  [{"loc":[5,6],"terminateAfterAction":false,"timeLeft":4,"previousLoc":[5,5]},"l"],
  [{"loc":[4,6],"terminateAfterAction":false,"timeLeft":3,"previousLoc":[5,6]},"u"],
  [{"loc":[4,7],"terminateAfterAction":false,"timeLeft":2,"previousLoc":[4,6],"timeAtRestaurant":0},"l"],
  [{"loc":[4,7],"terminateAfterAction":true,"timeLeft":2,"previousLoc":[4,7],"timeAtRestaurant":1},"l"]
];
///

// Prior on agent's utility function
var priorUtility = function() {
  var utilityValues = [-10, 0, 10, 20, 30, 40];
  // with no discounting, delayed utilities are ommitted
  var donut = [uniformDraw(utilityValues), 0];
  var veg = [uniformDraw(utilityValues), 0];
  return {
    'Donut N': donut,
    'Donut S': donut,
    'Veg': veg,
    'Noodle': [-10, -10],
    'timeCost': -.01
  };
};

// We assume no discounting (so *sophisticated* has no effect here)
var priorDiscounting = function() {
  return {
    discount: 0,
    sophisticatedOrNaive: 'sophisticated'
  };
};

var priorAlpha = function(){ return uniformDraw([0.1, 10, 100, 1000]); };
var prior = {
  utility: priorUtility,
  discounting: priorDiscounting,
  alpha: priorAlpha
};

// Get world and observations
var world = mdp.world;

print('Prior and posterior after observing Naive path');
var posteriorNaive = getPosterior(world, prior, naiveTrajectory);
displayResults(getPosterior(world, prior, []), posteriorNaive);

print('Prior and posterior after observing Sophisticated path');
var posteriorSophisticated = getPosterior(world, prior, sophisticatedTrajectory);
displayResults(getPosterior(world, prior, []), posteriorSophisticated);
~~~~

The graphs show two important results:

1. For the Naive path, the agent is inferred to prefer Donut, while for the Sophisticated path, Veg is inferred. In both cases, the inference fits with where the agent ends up.

2. High values for $$\alpha$$ are ruled out in each case, showing that the model explains the behavior in terms of noise.

What happens if we observe the agent taking the Naive path *repeatedly*? While noise is needed to explain the agent's path, too much noise is inconsistent with taking an identical path repeatedly. This is confirmed in the results below:

~~~~
///fold: Prior is same as above
var restaurantHyperbolicInfer = getRestaurantHyperbolicInfer();
var getPosterior = restaurantHyperbolicInfer.getPosterior;

var displayResults = function(priorDist, posteriorDist) {

  var priorUtility = priorDist.MAP().val.utility;
  print('Prior highest-probability utility for Veg: ' + priorUtility['Veg']
        + '. Donut: ' + priorUtility['Donut N'] + ' \n');

  var posteriorUtility = posteriorDist.MAP().val.utility;
  print('Posterior highest-probability utility for Veg: '
        + posteriorUtility['Veg'] + '. Donut: ' + posteriorUtility['Donut N']
        + ' \n');

  var getPriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(priorDist, label);
    return Math.exp(dist.score(x));
  };

  var getPosteriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(posteriorDist, label);
    return Math.exp(dist.score(x));
  };

  var vegMinusDonutPriorDataTable = map(
    function(x) {
      return {
        vegMinusDonut: x,
        probability: getPriorProb({ vegMinusDonut: x }),
        distribution: 'prior'
      };
    },
    [-50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50]);

  var vegMinusDonutPosteriorDataTable = map(
    function(x){
      return {
        vegMinusDonut: x,
        probability: getPosteriorProb({ vegMinusDonut: x }),
        distribution: 'posterior'
      };
    },
    [-50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50]);

  var vegMinusDonutDataTable = append(vegMinusDonutPriorDataTable,
                                      vegMinusDonutPosteriorDataTable);

  viz.bar(vegMinusDonutDataTable, { groupBy: 'distribution' });

  var alphaPriorDataTable = map(
    function(x){
      return {
        alpha: x,
        probability: getPriorProb({alpha: x}),
        distribution: 'prior'
      };
    },
    [0.1, 10, 100, 1000]);

  var alphaPosteriorDataTable = map(
    function(x){
      return {
        alpha: x,
        probability: getPosteriorProb({alpha: x}),
        distribution: 'posterior'
      };
    },
    [0.1, 10, 100, 1000]);

  var alphaDataTable = append(alphaPriorDataTable,
                              alphaPosteriorDataTable);

  viz.bar(alphaDataTable, { groupBy: 'distribution' });
};

// Prior on agent's utility function
var priorUtility = function() {
  var utilityValues = [-10, 0, 10, 20, 30, 40];
  // with no discounting, delayed utilities are ommitted
  var donut = [uniformDraw(utilityValues), 0];
  var veg = [uniformDraw(utilityValues), 0];
  return {
    'Donut N': donut,
    'Donut S': donut,
    'Veg': veg,
    'Noodle': [-10, -10],
    'timeCost': -.01
  };
};

// We assume no discounting (so *sophisticated* has no effect here)
var priorDiscounting = function(){
  return {
    discount: 0,
    sophisticatedOrNaive: 'sophisticated'
  };
};

var priorAlpha = function(){
  return uniformDraw([0.1, 10, 100, 1000]);
};
var prior = {
  utility: priorUtility,
  discounting: priorDiscounting,
  alpha: priorAlpha
};

var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});

var naiveTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"u"],
  [{"loc":[3,4],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"u"],
  [{"loc":[3,5],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[3,4]},"l"],
  [{"loc":[2,5],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[3,5],"timeAtRestaurant":0},"l"],
  [{"loc":[2,5],"terminateAfterAction":true,"timeLeft":6,"previousLoc":[2,5],"timeAtRestaurant":1},"l"]
];
///

var numberRepeats = 2; // with 2 repeats, we condition a total of 3 times
var posteriorNaive = getPosterior(mdp.world, prior, naiveTrajectory, numberRepeats);
print('Prior and posterior after conditioning 3 times on Naive path');
displayResults(getPosterior(mdp.world, prior, []), posteriorNaive);
~~~~

--------

### Model that includes discounting: jointly infer discounting, preferences, softmax noise

Our inference model now has the optimal agent as a special case but also includes time-inconsistent agents. This model jointly infers the discounting behavior, the agent's utilities and the softmax noise.

We show two different posteriors. The first is after conditioning on the Naive path (as above). In the second, we imagine that we have observed the agent taking the same path on multiple occasions (three times) and we condition on this.

~~~~
///fold:
var restaurantHyperbolicInfer = getRestaurantHyperbolicInfer();
var getPosterior = restaurantHyperbolicInfer.getPosterior;

var displayResults = function(priorDist, posteriorDist) {

  var priorUtility = priorDist.MAP().val.utility;
  print('Prior highest-probability utility for Veg: ' + priorUtility['Veg']
        + '. Donut: ' + priorUtility['Donut N'] + ' \n');

  var posteriorUtility = posteriorDist.MAP().val.utility;
  print('Posterior highest-probability utility for Veg: '
        + posteriorUtility['Veg'] + '. Donut: ' + posteriorUtility['Donut N']
        + ' \n');

  var getPriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(priorDist, label);
    return Math.exp(dist.score(x));
  };

  var getPosteriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(posteriorDist, label);
    return Math.exp(dist.score(x));
  };

  var sophisticationPriorDataTable = map(
    function(x) {
      return {
        sophisticatedOrNaive: x,
        probability: getPriorProb({ sophisticatedOrNaive: x }),
        distribution: 'prior'
      };
    },
    ['naive', 'sophisticated']);

  var sophisticationPosteriorDataTable = map(
    function(x) {
      return {
        sophisticatedOrNaive: x,
        probability: getPosteriorProb({ sophisticatedOrNaive: x }),
        distribution: 'posterior'
      };
    },
    ['naive', 'sophisticated']);

  var sophisticatedOrNaiveDataTable = append(sophisticationPosteriorDataTable,
                                             sophisticationPriorDataTable);

  viz.bar(sophisticatedOrNaiveDataTable, { groupBy: 'distribution' });

  var vegMinusDonutPriorDataTable = map(
    function(x) {
      return {
        vegMinusDonut: x,
        probability: getPriorProb({ vegMinusDonut: x }),
        distribution: 'prior'
      };
    },
    [-10, 0, 10, 20, 30, 40, 50, 60, 70]);

  var vegMinusDonutPosteriorDataTable = map(
    function(x) {
      return {
        vegMinusDonut: x,
        probability: getPosteriorProb({ vegMinusDonut: x }),
        distribution: 'posterior'
      };
    },
    [-10, 0, 10, 20, 30, 40, 50, 60, 70]);

  var vegMinusDonutDataTable = append(vegMinusDonutPriorDataTable,
                                      vegMinusDonutPosteriorDataTable);

  viz.bar(vegMinusDonutDataTable, { groupBy: 'distribution' });

  var donutTemptingPriorDataTable = map(
    function(x) {
      return {
        donutTempting: x,
        probability: getPriorProb({donutTempting: x}),
        distribution: 'prior'
      };
    },
    [true, false]);

  var donutTemptingPosteriorDataTable = map(
    function(x){
      return {
        donutTempting: x,
        probability: getPosteriorProb({ donutTempting: x }),
        distribution: 'posterior'
      };
    },
    [true, false]);

  var donutTemptingDataTable = append(donutTemptingPriorDataTable,
                                      donutTemptingPosteriorDataTable);

  viz.bar(donutTemptingDataTable, { groupBy: 'distribution' });

  var alphaPriorDataTable = map(
    function(x){
      return {
        alpha: x,
        probability: getPriorProb({alpha: x}),
        distribution: 'prior'
      };
    },
    [0.1, 10, 1000]);

  var alphaPosteriorDataTable = map(
    function(x){
      return {
        alpha: x,
        probability: getPosteriorProb({ alpha: x }),
        distribution: 'posterior'
      };
    },
    [0.1, 10, 1000]);

  var alphaDataTable = append(alphaPriorDataTable,
                              alphaPosteriorDataTable);

  viz.bar(alphaDataTable, { groupBy: 'distribution' });
};

var naiveTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"u"],
  [{"loc":[3,4],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"u"],
  [{"loc":[3,5],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[3,4]},"l"],
  [{"loc":[2,5],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[3,5],"timeAtRestaurant":0},"l"],
  [{"loc":[2,5],"terminateAfterAction":true,"timeLeft":6,"previousLoc":[2,5],"timeAtRestaurant":1},"l"]
];

var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});
///

// Prior on agent's utility function. We fix the delayed utilities
// to make inference faster
var priorUtility = function() {
  var utilityValues =  [-10, 0, 10, 20, 30];
  var donut = [uniformDraw(utilityValues), -10];
  var veg = [uniformDraw(utilityValues), 20];
  return {
    'Donut N': donut,
    'Donut S': donut,
    'Veg': veg,
    'Noodle': [-10, -10],
    'timeCost': -.01
  };
};

var priorDiscounting = function() {
  return {
    discount: uniformDraw([0, 1]),
    sophisticatedOrNaive: uniformDraw(['naive','sophisticated'])
  };
};
var priorAlpha = function(){
  return uniformDraw([.1, 10, 1000]);
};
var prior = {
  utility: priorUtility,
  discounting: priorDiscounting,
  alpha: priorAlpha
};

// Get world and observations
var world = mdp.world;

var posterior = getPosterior(world, prior, naiveTrajectory);
print('Prior and posterior after observing Naive path');
displayResults(getPosterior(world, prior, []), posterior);

print('Prior and posterior after observing Naive path three times');
var numberRepeats = 2;
displayResults(getPosterior(world, prior, []),
               getPosterior(world, prior, naiveTrajectory, numberRepeats));
~~~~

Conditioning on the Naive path once, the probabilities of the agent being Naive and of `donutTempting` both go up. However, the probability of high softmax noise also goes up. In terms of preferences, we rule out a strong preference for Veg and slightly reduce a preference for Donut. So if the agent were Naive, tempted by Donut and with very low noise, our inference would not place most of the posterior on this explanation. There are two reasons for this. First, this agent is unlikely in the prior. Second, the explanation of the behavior in terms of noise is plausible. (In our Gridworld setup, we don't allow the agent to backtrack to the previous state. This means there are few cases where a softmax noisy agent would behavior differently than a low noise one.). Conditioning on the same Naive path three times makes the explanation in terms of noise much less plausible: the agent would makes the same "mistake" three times and makes no other mistakes. (The results for the Sophisticated path are similar.)

In summary, if we observe the agent repeatedly take the Naive path, the "Optimal Model" explains this in terms of a preference for Donut and significant softmax noise (explaining why the agent takes Donut North over Donut South). The "Discounting Model" is similar to the Optimal Model when it observes the Naive path *once*. However, observing it multiple times, it infers that the agent has low noise and an overall preference for Veg.

------

### Preferences for the two Donut Store branches can vary

Another explanation of the Naive path is that the agent has a preference for the "Donut N" branch of the Donut Store over the "Donut S" branch. Maybe this branch is better run or has more space. If we add this to our set of possible preferences, inference changes significantly.

To speed up inference, we use a fixed assumption that the agent is Naive. There are three explanations of the agent's path:

1. Softmax noise: measured by $$\alpha$$
2. The agent is Naive and tempted by Donut: measured by `discount` and `donutTempting`
3. The agent prefers Donut N to Donut S: measured by `donutNGreaterDonutS` (i.e. Donut N's utility is greater than Donut S's).

These three can also be combined to explain the behavior.

~~~~
///fold:
var restaurantHyperbolicInfer = getRestaurantHyperbolicInfer();
var getPosterior = restaurantHyperbolicInfer.getPosterior;

var naiveTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"u"],
  [{"loc":[3,4],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"u"],
  [{"loc":[3,5],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[3,4]},"l"],
  [{"loc":[2,5],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[3,5],"timeAtRestaurant":0},"l"],
  [{"loc":[2,5],"terminateAfterAction":true,"timeLeft":6,"previousLoc":[2,5],"timeAtRestaurant":1},"l"]
];

var displayResults = function(priorDist, posteriorDist) {

  var priorUtility = priorDist.MAP().val.utility;
  print('Prior highest-probability utility for Veg: ' + priorUtility['Veg']
        + '. Donut: ' + priorUtility['Donut N'] + ' \n');

  var posteriorUtility = posteriorDist.MAP().val.utility;
  print('Posterior highest-probability utility for Veg: '
        + posteriorUtility['Veg'] + '. Donut: ' + posteriorUtility['Donut N']
        + ' \n');

  var getPriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(priorDist, label);
    return Math.exp(dist.score(x));
  };

  var getPosteriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(posteriorDist, label);
    return Math.exp(dist.score(x));
  };

  var alphaPriorDataTable = map(
    function(x) {
      return {
        alpha: x,
        probability: getPriorProb({alpha: x}),
        distribution: 'prior'
      };
    },
    [0.1, 100, 1000]);

  var alphaPosteriorDataTable = map(
    function(x) {
      return {
        alpha: x,
        probability: getPosteriorProb({alpha: x}),
        distribution: 'posterior'
      };
    },
    [0.1, 100, 1000]);

  var alphaDataTable = append(alphaPriorDataTable,
                              alphaPosteriorDataTable);

  viz.bar(alphaDataTable, { groupBy: 'distribution' });

  var donutTemptingPriorDataTable = map(
    function(x) {
      return {
        donutTempting: x,
        probability: getPriorProb({ donutTempting: x }),
        distribution: 'prior'
      };
    },
    [true, false]);

  var donutTemptingPosteriorDataTable = map(
    function(x) {
      return {
        donutTempting: x,
        probability: getPosteriorProb({ donutTempting: x }),
        distribution: 'posterior'
      };
    },
    [true, false]);

  var donutTemptingDataTable = append(donutTemptingPriorDataTable,
                                      donutTemptingPosteriorDataTable);

  viz.bar(donutTemptingDataTable, { groupBy: 'distribution' });

  var discountPriorDataTable = map(
    function(x) {
      return {
        discount: x,
        probability: getPriorProb({ discount: x }),
        distribution: 'prior'
      };
    },
    [0, 1]);

  var discountPosteriorDataTable = map(
    function(x) {
      return {
        discount: x,
        probability: getPosteriorProb({ discount: x }),
        distribution: 'posterior'
      };
    },
    [0, 1]);

  var discountDataTable = append(discountPriorDataTable,
                                 discountPosteriorDataTable);

  viz.bar(discountDataTable, { groupBy: 'distribution' });

  var donutNvsSPriorDataTable = map(
    function(x) {
      return {
        donutNGreaterDonutS: x,
        probability: getPriorProb({ donutNGreaterDonutS: x }),
        distribution: 'prior'
      };
    },
    [false, true]);

  var donutNvsSPosteriorDataTable = map(
    function(x) {
      return {
        donutNGreaterDonutS: x,
        probability: getPosteriorProb({ donutNGreaterDonutS: x }),
        distribution: 'posterior'
      };
    },
    [false, true]);

  var donutNvsSDataTable = append(donutNvsSPriorDataTable,
                                  donutNvsSPosteriorDataTable);

  viz.bar(donutNvsSDataTable, { groupBy: 'distribution' });
};

var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});
///

// Prior on agent's utility function
var priorUtility = function() {
  var utilityValues = [-10, 0, 10, 20];
  return {
    'Donut N': [uniformDraw(utilityValues), -10],
    'Donut S': [uniformDraw(utilityValues), -10],
    'Veg': [20, uniformDraw(utilityValues)],
    'Noodle': [-10, -10],
    'timeCost': -.01
  };
};

var priorDiscounting = function() {
  return {
    discount: uniformDraw([0, 1]),
    sophisticatedOrNaive: 'naive'
  };
};
var priorAlpha = function(){
  return uniformDraw([.1, 100, 1000]);
};
var prior = {
  utility: priorUtility,
  discounting: priorDiscounting,
  alpha: priorAlpha
};

// Get world and observations
var posterior = getPosterior(mdp.world, prior, naiveTrajectory);
displayResults(getPosterior(mdp.world, prior, []), posterior);
~~~~

The explanation in terms of Donut North being preferred does well in the posterior. This is because the discounting explanation (even assuming the agent is Naive) is unlikely a priori (due to our simple uniform priors on utilities and discounting). While high noise is more plausible a priori, the noise explanation still needs to posit a low probability series of events.

We see a similar result if we enrich the set of possible utilities for the Sophisticated path. This time, we allow the `timeCost`, i.e. the cost for taking a single timestep, to be positive. This means the agent prefers to spend as much time as possible moving around before reaching a restaurant. Here are the results:

Observe the sophisticated path with possibly positive timeCost:

~~~~
///fold:
var restaurantHyperbolicInfer = getRestaurantHyperbolicInfer();
var getPosterior = restaurantHyperbolicInfer.getPosterior;

var displayResults = function(priorDist, posteriorDist) {

  var priorUtility = priorDist.MAP().val.utility;
  print('Prior highest-probability utility for Veg: ' + priorUtility['Veg']
        + '. Donut: ' + priorUtility['Donut N'] + ' \n');

  var posteriorUtility = posteriorDist.MAP().val.utility;
  print('Posterior highest-probability utility for Veg: '
        + posteriorUtility['Veg'] + '. Donut: ' + posteriorUtility['Donut N']
        + ' \n');

  var getPriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(priorDist, label);
    return Math.exp(dist.score(x));
  };

  var getPosteriorProb = function(x) {
    var label = _.keys(x)[0];
    var dist = getMarginalObject(posteriorDist, label);
    return Math.exp(dist.score(x));
  };

  var alphaPriorDataTable = map(
    function(x) {
      return {
        alpha: x,
        probability: getPriorProb({alpha: x}),
        distribution: 'prior'
      };
    },
    [0.1, 100, 1000]);

  var alphaPosteriorDataTable = map(
    function(x) {
      return {
        alpha: x,
        probability: getPosteriorProb({alpha: x}),
        distribution: 'posterior'
      };
    },
    [0.1, 100, 1000]);

  var alphaDataTable = append(alphaPriorDataTable,
                              alphaPosteriorDataTable);

  viz.bar(alphaDataTable, { groupBy: 'distribution' });

  var donutTemptingPriorDataTable = map(
    function(x) {
      return {
        donutTempting: x,
        probability: getPriorProb({ donutTempting: x }),
        distribution: 'prior'
      };
    },
    [true, false]);

  var donutTemptingPosteriorDataTable = map(
    function(x) {
      return {
        donutTempting: x,
        probability: getPosteriorProb({ donutTempting: x }),
        distribution: 'posterior'
      };
    },
    [true, false]);

  var donutTemptingDataTable = append(donutTemptingPriorDataTable,
                                      donutTemptingPosteriorDataTable);

  viz.bar(donutTemptingDataTable, { groupBy: 'distribution' });

  var discountPriorDataTable = map(
    function(x){
      return {
        discount: x,
        probability: getPriorProb({ discount: x }),
        distribution: 'prior'
      };
    },
    [0, 1]);

  var discountPosteriorDataTable = map(
    function(x){
      return {
        discount: x,
        probability: getPosteriorProb({ discount: x }),
        distribution: 'posterior'
      };
    },
    [0, 1]);

  var discountDataTable = append(discountPriorDataTable,
                                 discountPosteriorDataTable);

  viz.bar(discountDataTable, { groupBy: 'distribution' });

  var timeCostPriorDataTable = map(
    function(x) {
      return {
        timeCost: x,
        probability: getPriorProb({ timeCost: x }),
        distribution: 'prior'
      };
    },
    [-0.01, 0.1, 1]);

  var timeCostPosteriorDataTable = map(
    function(x) {
      return {
        timeCost: x,
        probability: getPosteriorProb({ timeCost: x }),
        distribution: 'posterior'
      };
    },
    [-0.01, 0.1, 1]);

  var timeCostDataTable = append(timeCostPriorDataTable,
                                 timeCostPosteriorDataTable);

  viz.bar(timeCostDataTable, { groupBy: 'distribution' });
};

var sophisticatedTrajectory = [
  [{"loc":[3,1],"terminateAfterAction":false,"timeLeft":11},"u"],
  [{"loc":[3,2],"terminateAfterAction":false,"timeLeft":10,"previousLoc":[3,1]},"u"],
  [{"loc":[3,3],"terminateAfterAction":false,"timeLeft":9,"previousLoc":[3,2]},"r"],
  [{"loc":[4,3],"terminateAfterAction":false,"timeLeft":8,"previousLoc":[3,3]},"r"],
  [{"loc":[5,3],"terminateAfterAction":false,"timeLeft":7,"previousLoc":[4,3]},"u"],
  [{"loc":[5,4],"terminateAfterAction":false,"timeLeft":6,"previousLoc":[5,3]},"u"],
  [{"loc":[5,5],"terminateAfterAction":false,"timeLeft":5,"previousLoc":[5,4]},"u"],
  [{"loc":[5,6],"terminateAfterAction":false,"timeLeft":4,"previousLoc":[5,5]},"l"],
  [{"loc":[4,6],"terminateAfterAction":false,"timeLeft":3,"previousLoc":[5,6]},"u"],
  [{"loc":[4,7],"terminateAfterAction":false,"timeLeft":2,"previousLoc":[4,6],"timeAtRestaurant":0},"l"],
  [{"loc":[4,7],"terminateAfterAction":true,"timeLeft":2,"previousLoc":[4,7],"timeAtRestaurant":1},"l"]
];

var ___ = ' ';
var DN = { name : 'Donut N' };
var DS = { name : 'Donut S' };
var V = { name : 'Veg' };
var N = { name : 'Noodle' };

var grid = [
  ['#', '#', '#', '#',  V , '#'],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', DN , ___, '#', ___],
  ['#', '#', '#', ___, '#', ___],
  ['#', '#', '#', ___, ___, ___],
  ['#', '#', '#', ___, '#',  N ],
  [___, ___, ___, ___, '#', '#'],
  [DS , '#', '#', ___, '#', '#']
];

var mdp = makeGridWorldMDP({
  grid,
  noReverse: true,
  maxTimeAtRestaurant: 2,
  start: [3, 1],
  totalTime: 11
});
///

// Prior on agent's utility function
var priorUtility = function() {
  var utilityValues =  [-10, 0, 10, 20, 30];
  var donut = [uniformDraw(utilityValues), -10]
  var veg = [uniformDraw(utilityValues), 20];
  return {
    'Donut N': donut,
    'Donut S': donut,
    'Veg': veg,
    'Noodle': [-10, -10],
    'timeCost': uniformDraw([-0.01, 0.1, 1])
  };
};

var priorDiscounting = function() {
  return {
    discount: uniformDraw([0, 1]),
    sophisticatedOrNaive: 'sophisticated'
  };
};
var priorAlpha = function(){
  return uniformDraw([0.1, 100, 1000]);
};
var prior = {
  utility: priorUtility,
  discounting: priorDiscounting,
  alpha: priorAlpha
};

var posterior = getPosterior(mdp.world, prior, sophisticatedTrajectory);
displayResults(getPosterior(mdp.world, prior, []), posterior);
~~~~

Next chapter: [Multi-agent models](/chapters/7-multi-agent.html)


7-multi-agent.md

---
layout: chapter
title: Multi-agent models
description: Schelling coordination games, tic-tac-toe, and a simple natural-language example.
is_section: true
---

In this chapter, we will look at models that involve multiple agents reasoning about each other.
This chapter is based on reft:stuhlmueller2013reasoning.

## Schelling coordination games

We start with a simple [Schelling coordination game](http://lesswrong.com/lw/dc7/nash_equilibria_and_schelling_points/). Alice and Bob are trying to meet up but have lost their phones and have no way to contact each other. There are two local bars: the popular bar and the unpopular one.

Let's first consider how Alice would choose a bar (if she was not taking Bob into account):

~~~~
var locationPrior = function() {
  if (flip(.55)) {
    return 'popular-bar';
  } else {
    return 'unpopular-bar';
  }
};

var alice = function() {
  return Infer({ model() {
    var myLocation = locationPrior();
    return myLocation;
  }});
};

viz(alice());
~~~~

But Alice wants to be at the same bar as Bob. We extend our model of Alice to include this:

~~~~
var locationPrior = function() {
  if (flip(.55)) {
    return 'popular-bar';
  } else {
    return 'unpopular-bar';
  }
};

var alice = function() {
  return Infer({ model() {
    var myLocation = locationPrior();
    var bobLocation = sample(bob());
    condition(myLocation === bobLocation);
    return myLocation;
  }});
};

var bob = function() {
  return Infer({ model() {
    var myLocation = locationPrior();
    return myLocation;
  }});
};

viz(alice());
~~~~

Now Bob and Alice are thinking recursively about each other. We add caching (to avoid repeated computations) and a depth parameter (to avoid infinite recursion):

~~~~
var locationPrior = function() {
  if (flip(.55)) {
    return 'popular-bar';
  } else {
    return 'unpopular-bar';
  }
}

var alice = dp.cache(function(depth) {
  return Infer({ model() {
    var myLocation = locationPrior();
    var bobLocation = sample(bob(depth - 1));
    condition(myLocation === bobLocation);
    return myLocation;
  }});
});

var bob = dp.cache(function(depth) {
  return Infer({ model() {
    var myLocation = locationPrior();
    if (depth === 0) {
      return myLocation;
    } else {
      var aliceLocation = sample(alice(depth));
      condition(myLocation === aliceLocation);
      return myLocation;
    }
  }});
});

viz(alice(10));
~~~~

>**Exercise**: Change the example to the setting where Bob wants to avoid Alice instead of trying to meet up with her, and Alice knows this. How do the predictions change as the reasoning depth grows? How would you model the setting where Alice doesn't know that Bob wants to avoid her?

>**Exercise**: Would any of the answers to the previous exercise change if recursive reasoning could terminate not just at a fixed depth, but also at random?

## Game playing

We'll look at the two-player game tic-tac-toe:

<img src="/assets/img/tic-tac-toe-game-1.svg"/>

>*Figure 1:* Tic-tac-toe. (Image source: [Wikipedia](https://en.wikipedia.org/wiki/Tic-tac-toe#/media/File:Tic-tac-toe-game-1.svg))

Let's start with a prior on moves:

~~~~
var isValidMove = function(state, move) {
  return state[move.x][move.y] === '?';
};

var movePrior = dp.cache(function(state){
  return Infer({ model() {
    var move = {
      x: randomInteger(3),
      y: randomInteger(3)
    };
    condition(isValidMove(state, move));
    return move;
  }});
});

var startState = [
  ['?', 'o', '?'],
  ['?', 'x', 'x'],
  ['?', '?', '?']
];

viz.table(movePrior(startState));
~~~~

Now let's add a deterministic transition function:

~~~~
///fold: isValidMove, movePrior
var isValidMove = function(state, move) {
  return state[move.x][move.y] === '?';
};

var movePrior = dp.cache(function(state){
  return Infer({ model() {
    var move = {
      x: randomInteger(3),
      y: randomInteger(3)
    };
    condition(isValidMove(state, move));
    return move;
  }});
});
///

var assign = function(obj, k, v) {
  var newObj = _.clone(obj);
  return Object.defineProperty(newObj, k, {value: v})
};

var transition = function(state, move, player) {
  var newRow = assign(state[move.x], move.y, player);
  return assign(state, move.x, newRow);
};

var startState = [
  ['?', 'o', '?'],
  ['?', 'x', 'x'],
  ['?', '?', '?']
];

transition(startState, {x: 1, y: 0}, 'o');
~~~~

We need to be able to check if a player has won:

~~~~
///fold: movePrior, transition
var isValidMove = function(state, move) {
  return state[move.x][move.y] === '?';
};

var movePrior = dp.cache(function(state){
  return Infer({ model() {
    var move = {
      x: randomInteger(3),
      y: randomInteger(3)
    };
    condition(isValidMove(state, move));
    return move;
  }});
});

var assign = function(obj, k, v) {
  var newObj = _.clone(obj);
  return Object.defineProperty(newObj, k, {value: v})
};

var transition = function(state, move, player) {
  var newRow = assign(state[move.x], move.y, player);
  return assign(state, move.x, newRow);
};
///

var diag1 = function(state) {
  return mapIndexed(function(i, x) {return x[i];}, state);
};

var diag2 = function(state) {
  var n = state.length;
  return mapIndexed(function(i, x) {return x[n - (i + 1)];}, state);
};

var hasWon = dp.cache(function(state, player) {
  var check = function(xs){
    return _.countBy(xs)[player] == xs.length;
  };
  return any(check, [
    state[0], state[1], state[2], // rows
    map(first, state), map(second, state), map(third, state), // cols
    diag1(state), diag2(state) // diagonals
  ]);
});

var startState = [
  ['?', 'o', '?'],
  ['x', 'x', 'x'],
  ['?', '?', '?']
];

hasWon(startState, 'x');
~~~~

Now let's implement an agent that chooses a single action, but can't plan ahead:

~~~~
///fold: movePrior, transition, hasWon
var isValidMove = function(state, move) {
  return state[move.x][move.y] === '?';
};

var movePrior = dp.cache(function(state){
  return Infer({ model() {
    var move = {
      x: randomInteger(3),
      y: randomInteger(3)
    };
    condition(isValidMove(state, move));
    return move;
  }});
});

var assign = function(obj, k, v) {
  var newObj = _.clone(obj);
  return Object.defineProperty(newObj, k, {value: v})
};

var transition = function(state, move, player) {
  var newRow = assign(state[move.x], move.y, player);
  return assign(state, move.x, newRow);
};

var diag1 = function(state) {
  return mapIndexed(function(i, x) {return x[i];}, state);
};

var diag2 = function(state) {
  var n = state.length;
  return mapIndexed(function(i, x) {return x[n - (i + 1)];}, state);
};

var hasWon = dp.cache(function(state, player) {
  var check = function(xs){
    return _.countBy(xs)[player] == xs.length;
  };
  return any(check, [
    state[0], state[1], state[2], // rows
    map(first, state), map(second, state), map(third, state), // cols
    diag1(state), diag2(state) // diagonals
  ]);
});
///
var isDraw = function(state) {
  return !hasWon(state, 'x') && !hasWon(state, 'o');
};

var utility = function(state, player) {
  if (hasWon(state, player)) {
    return 10;
  } else if (isDraw(state)) {
    return 0;
  } else {
    return -10;
  }
};

var act = dp.cache(function(state, player) {
  return Infer({ model() {
    var move = sample(movePrior(state));
    var eu = expectation(Infer({ model() {
      var outcome = transition(state, move, player);
      return utility(outcome, player);
    }}));
    factor(eu);
    return move;
  }});
});

var startState = [
  ['o', 'o', '?'],
  ['?', 'x', 'x'],
  ['?', '?', '?']
];

viz.table(act(startState, 'x'));
~~~~

And now let's include planning:

~~~~
///fold: movePrior, transition, hasWon, utility, isTerminal
var isValidMove = function(state, move) {
  return state[move.x][move.y] === '?';
};

var movePrior = dp.cache(function(state){
  return Infer({ model() {
    var move = {
      x: randomInteger(3),
      y: randomInteger(3)
    };
    condition(isValidMove(state, move));
    return move;
  }});
});

var assign = function(obj, k, v) {
  var newObj = _.clone(obj);
  return Object.defineProperty(newObj, k, {value: v})
};

var transition = function(state, move, player) {
  var newRow = assign(state[move.x], move.y, player);
  return assign(state, move.x, newRow);
};

var diag1 = function(state) {
  return mapIndexed(function(i, x) {return x[i];}, state);
};

var diag2 = function(state) {
  var n = state.length;
  return mapIndexed(function(i, x) {return x[n - (i + 1)];}, state);
};

var hasWon = dp.cache(function(state, player) {
  var check = function(xs){
    return _.countBy(xs)[player] == xs.length;
  };
  return any(check, [
    state[0], state[1], state[2], // rows
    map(first, state), map(second, state), map(third, state), // cols
    diag1(state), diag2(state) // diagonals
  ]);
});

var isDraw = function(state) {
  return !hasWon(state, 'x') && !hasWon(state, 'o');
};

var utility = function(state, player) {
  if (hasWon(state, player)) {
    return 10;
  } else if (isDraw(state)) {
    return 0;
  } else {
    return -10;
  }
};

var isComplete = function(state) {
  return all(
    function(x){
      return x != '?';
    },
    _.flatten(state));
}

var isTerminal = function(state) {
  return hasWon(state, 'x') || hasWon(state, 'o') || isComplete(state);
};
///

var otherPlayer = function(player) {
  return (player === 'x') ? 'o' : 'x';
};

var act = dp.cache(function(state, player) {
  return Infer({ model() {
    var move = sample(movePrior(state));
    var eu = expectation(Infer({ model() {
      var outcome = simulate(state, move, player);
      return utility(outcome, player);
    }}));
    factor(eu);
    return move;
  }});
});

var simulate = function(state, action, player) {
  var nextState = transition(state, action, player);
  if (isTerminal(nextState)) {
    return nextState;
  } else {
    var nextPlayer = otherPlayer(player);
    var nextAction = sample(act(nextState, nextPlayer));
    return simulate(nextState, nextAction, nextPlayer);
  }
};

var startState = [
  ['o', '?', '?'],
  ['?', '?', 'x'],
  ['?', '?', '?']
];

var actDist = act(startState, 'o');

viz.table(actDist);
~~~~

## Language understanding

<!-- TODO text needs more elaboration or some links to papers or online content -->

A model of pragmatic language interpretation: The speaker chooses a sentence conditioned on the listener inferring the intended state of the world when hearing this sentence; the listener chooses an interpretation conditioned on the speaker selecting the given utterance when intending this meaning.

Literal interpretation:

~~~~
var statePrior = function() {
  return uniformDraw([0, 1, 2, 3]);
};

var literalMeanings = {
  allSprouted: function(state) { return state === 3; },
  someSprouted: function(state) { return state > 0; },
  noneSprouted: function(state) { return state === 0; }
};

var sentencePrior = function() {
  return uniformDraw(['allSprouted', 'someSprouted', 'noneSprouted']);
};

var literalListener = function(sentence) {
  return Infer({ model() {
    var state = statePrior();
    var meaning = literalMeanings[sentence];
    condition(meaning(state));
    return state;
  }});
};

viz(literalListener('someSprouted'));
~~~~

A pragmatic speaker, thinking about the literal listener:

~~~~
var alpha = 2;

///fold: statePrior, literalMeanings, sentencePrior
var statePrior = function() {
  return uniformDraw([0, 1, 2, 3]);
};

var literalMeanings = {
  allSprouted: function(state) { return state === 3; },
  someSprouted: function(state) { return state > 0; },
  noneSprouted: function(state) { return state === 0; }
};

var sentencePrior = function() {
  return uniformDraw(['allSprouted', 'someSprouted', 'noneSprouted']);
};
///

var literalListener = function(sentence) {
  return Infer({ model() {
    var state = statePrior();
    var meaning = literalMeanings[sentence];
    condition(meaning(state));
    return state;
  }});
};

var speaker = function(state) {
  return Infer({ model() {
    var sentence = sentencePrior();
    factor(alpha * literalListener(sentence).score(state));
    return sentence;
  }});
}

viz(speaker(3));
~~~~

Pragmatic listener, thinking about speaker:

~~~~
var alpha = 2;

///fold: statePrior, literalMeanings, sentencePrior
var statePrior = function() {
  return uniformDraw([0, 1, 2, 3]);
};

var literalMeanings = {
  allSprouted: function(state) { return state === 3; },
  someSprouted: function(state) { return state > 0; },
  noneSprouted: function(state) { return state === 0; }
};

var sentencePrior = function() {
  return uniformDraw(['allSprouted', 'someSprouted', 'noneSprouted']);
};
///

var literalListener = dp.cache(function(sentence) {
  return Infer({ model() {
    var state = statePrior();
    var meaning = literalMeanings[sentence];
    condition(meaning(state));
    return state;
  }});
});

var speaker = dp.cache(function(state) {
  return Infer({ model() {
    var sentence = sentencePrior();
    factor(alpha * literalListener(sentence).score(state));
    return sentence;
  }});
});

var listener = dp.cache(function(sentence) {
  return Infer({ model() {
    var state = statePrior();
    factor(speaker(state).score(sentence));
    return state;
  }});
});

viz(listener('someSprouted'));
~~~~

Next chapter: [How to use the WebPPL Agent Models library](/chapters/8-guide-library.html)


3-agents-as-programs.md

---
layout: chapter
title: "Agents as probabilistic programs"
description: "One-shot decision problems, expected utility, softmax choice and Monty Hall."
is_section: true
---

## Introduction

Our goal is to implement agents that compute rational *policies*. Policies are *plans* for achieving good outcomes in environments where:

- The agent makes a *sequence* of *distinct* choices, rather than choosing once.

- The environment is *stochastic* (or "random").

- Some features of the environment are initially *unknown* to the agent. (So the agent may choose to gain information in order to improve future decisions.)

This section begins with agents that solve the very simplest decision problems. These are trivial *one-shot* problems, where the agent selects a single action (not a sequence of actions). We use WebPPL to solve these problems in order to illustrate the core concepts that are necessary for the more complex problems in later chapters.

## One-shot decisions in a deterministic world

In a *one-shot decision problem* an agent makes a single choice between a set of *actions*, each of which has potentially distinct *consequences*. A rational agent chooses the action that is best in terms of his or her own preferences. Often, this depends not on the *action* itself being preferred, but only on its *consequences*.

For example, suppose Tom is choosing between restaurants and all he cares about is eating pizza. There's an Italian restaurant and a French restaurant. Tom would choose the French restaurant if it offered pizza. Since it does *not* offer pizza, Tom will choose the Italian.

Tom selects an action $$a \in A$$ from the set of all actions. The actions in this case are {"eat at Italian restaurant", "eat at French restaurant"}. The consequences of an action are represented by a transition function $$T \colon S \times A \to S$$ from state-action pairs to states. In our example, the relevant *state* is whether or not Tom eats pizza. Tom's preferences are represented by a real-valued utility function $$U \colon S \to \mathbb{R}$$, which indicates the relative goodness of each state.

Tom's *decision rule* is to take action $$a$$ that maximizes utility, i.e., the action

$$
{\arg \max}_{a \in A} U(T(s,a))
$$

In WebPPL, we can implement this utility-maximizing agent as a function `maxAgent` that takes a state $$s \in S$$ as input and returns an action. For Tom's choice between restaurants, we assume that the agent starts off in a state `"initialState"`, denoting whatever Tom does before going off to eat. The program directly translates the decision rule above using the higher-order function `argMax`.

~~~~
///fold: argMax
var argMax = function(f, ar){
  return maxWith(f, ar)[0]
};
///
// Choose to eat at the Italian or French restaurants
var actions = ['italian', 'french'];

var transition = function(state, action) {
  if (action === 'italian') {
    return 'pizza';
  } else {
    return 'steak frites';
  }
};

var utility = function(state) {
  if (state === 'pizza') {
    return 10;
  } else {
    return 0;
  }
};

var maxAgent = function(state) {
  return argMax(
    function(action) {
      return utility(transition(state, action));
    },
    actions);
};

print('Choice in initial state: ' + maxAgent('initialState'));
~~~~

>**Exercise**: Which parts of the code can you change in order to make the agent choose the French restaurant?

There is an alternative way to compute the optimal action for this problem. The idea is to treat choosing an action as an *inference* problem. The previous chapter showed how we can *infer* the probability that a coin landed Heads from the observation that two of three coins were Heads.

~~~~
var twoHeads = Infer({
  model() {
    var a = flip(0.5);
    var b = flip(0.5);
    var c = flip(0.5);
    condition(a + b + c === 2);
    return a;
  }
});

viz(twoHeads);
~~~~

The same inference machinery can compute the optimal action in Tom's decision problem. We sample random actions with `uniformDraw` and condition on the preferred outcome happening. Intuitively, we imagine observing the consequence we prefer (e.g. pizza) and then *infer* from this the action that caused this consequence.

This idea is known as "planning as inference" refp:botvinick2012planning. It also resembles the idea of "backwards chaining" in logical inference and planning. The `inferenceAgent` solves the same problem as `maxAgent`, but uses planning as inference:

~~~~
var actions = ['italian', 'french'];

var transition = function(state, action) {
  if (action === 'italian') {
    return 'pizza';
  } else {
    return 'steak frites';
  }
};

var inferenceAgent = function(state) {
  return Infer({
    model() {
      var action = uniformDraw(actions);
      condition(transition(state, action) === 'pizza');
      return action;
    }
  });
}

viz(inferenceAgent("initialState"));
~~~~

>**Exercise**: Change the agent's goals so that they choose the French restaurant.

## One-shot decisions in a stochastic world

In the previous example, the transition function from state-action pairs to states was *deterministic* and so described a deterministic world or environment. Moreover, the agent's actions were deterministic; Tom always chose the best action ("Italian"). In contrast, many examples in this tutorial will involve a *stochastic* world and a noisy "soft-max" agent.

Imagine that Tom is choosing between restaurants again. This time, Tom's preferences are about the overall quality of the meal. A meal can be "bad", "good" or "spectacular" and each restaurant has good nights and bad nights. The transition function now has type signature $$ T\colon S \times A \to \Delta S $$, where $$\Delta S$$ represents a distribution over states. Tom's decision rule is now to take the action $$a \in A$$ that has the highest *average* or *expected* utility, with the expectation $$\mathbb{E}$$ taken over the probability of different successor states $$s' \sim T(s,a)$$:

$$
\max_{a \in A} \mathbb{E}( U(T(s,a)) )
$$

To represent this in WebPPL, we extend `maxAgent` using the `expectation` function, which maps a distribution with finite support to its (real-valued) expectation:

~~~~
///fold: argMax
var argMax = function(f, ar){
  return maxWith(f, ar)[0]
};
///

var actions = ['italian', 'french'];

var transition = function(state, action) {
  var nextStates = ['bad', 'good', 'spectacular'];
  var nextProbs = (action === 'italian') ? [0.2, 0.6, 0.2] : [0.05, 0.9, 0.05];
  return categorical(nextProbs, nextStates);
};

var utility = function(state) {
  var table = {
    bad: -10,
    good: 6,
    spectacular: 8
  };
  return table[state];
};

var maxEUAgent = function(state) {
  var expectedUtility = function(action) {
    return expectation(Infer({
      model() {
        return utility(transition(state, action));
      }
    }));
  };
  return argMax(expectedUtility, actions);
};

maxEUAgent('initialState');
~~~~

>**Exercise**: Adjust the transition probabilities such that the agent chooses the Italian Restaurant.

The `inferenceAgent`, which uses the planning-as-inference idiom, can also be extended using `expectation`. Previously, the agent's action was conditioned on leading to the best consequence ("pizza"). This time, Tom is not aiming to choose the action most likely to have the best outcome. Instead, he wants the action with better outcomes on average. This can be represented in `inferenceAgent` by switching from a `condition` statement to a `factor` statement. The `condition` statement expresses a "hard" constraint on actions: actions that fail the condition are completely ruled out. The `factor` statement, by contrast, expresses a "soft" condition. Technically, `factor(x)` adds `x` to the unnormalized log-probability of the program execution within which it occurs.

To illustrate `factor`, consider the following variant of the `twoHeads` example above. Instead of placing a hard constraint on the total number of Heads outcomes, we give each setting of `a`, `b` and `c` a *score* based on the total number of heads. The score is highest when all three coins are Heads, but even the "all tails" outcomes is not ruled out completely.

~~~~
var softHeads = Infer({
  model() {
    var a = flip(0.5);
    var b = flip(0.5);
    var c = flip(0.5);
    factor(a + b + c);
    return a;
  }
});

viz(softHeads);
~~~~

As another example, consider the following short program:

~~~~
var dist = Infer({
  model() {
    var n = uniformDraw([0, 1, 2]);
    factor(n * n);
    return n;
  }
});

viz(dist);
~~~~

Without the `factor` statement, each value of the variable `n` has equal probability. Adding the `factor` statements adds `n*n` to the log-score of each value. To get the new probabilities induced by the `factor` statement we compute the normalizing constant given these log-scores. The resulting probability $$P(y=2)$$ is:

$$
P(y=2) = \frac {e^{2 \cdot 2}} { (e^{0 \cdot 0} + e^{1 \cdot 1} + e^{2 \cdot 2}) }
$$

Returning to our implementation as planning-as-inference for maximizing *expected* utility, we use a `factor` statement to implement soft conditioning:

~~~~
var actions = ['italian', 'french'];

var transition = function(state, action) {
  var nextStates = ['bad', 'good', 'spectacular'];
  var nextProbs = (action === 'italian') ? [0.2, 0.6, 0.2] : [0.05, 0.9, 0.05];
  return categorical(nextProbs, nextStates);
};

var utility = function(state) {
  var table = {
    bad: -10,
    good: 6,
    spectacular: 8
  };
  return table[state];
};

var alpha = 1;

var softMaxAgent = function(state) {
  return Infer({
    model() {

      var action = uniformDraw(actions);

      var expectedUtility = function(action) {
        return expectation(Infer({
          model() {
            return utility(transition(state, action));
          }
        }));
      };

      factor(alpha * expectedUtility(action));

      return action;
    }
  });
};

viz(softMaxAgent('initialState'));
~~~~

The `softMaxAgent` differs in two ways from the `maxEUAgent` above. First, it uses the planning-as-inference idiom. Second, it does not deterministically choose the action with maximal expected utility. Instead, it implements *soft* maximization, selecting actions with a probability that depends on their expected utility. Formally, let the agent's probability of choosing an action be $$C(a;s)$$ for $$a \in A$$ when in state $$s \in S$$. Then the *softmax* decision rule is:

$$
C(a; s) \propto e^{\alpha \mathbb{E}(U(T(s,a))) }
$$

The noise parameter $$\alpha$$ modulates between random choice $$(\alpha=0)$$ and the perfect maximization $$(\alpha = \infty)$$ of the `maxEUAgent`.

Since rational agents will *always* choose the best action, why consider softmax agents? One of the goals of this tutorial is to infer the preferences of agents (e.g. human beings) from their choices. People do not always choose the normatively rational actions. The softmax agent provides a simple, analytically tractable model of sub-optimal choice[^softmax], which has been tested empirically on human action selection refp:luce2005individual. Moreover, it has been used extensively in Inverse Reinforcement Learning as a model of human errors refp:kim2014inverse, refp:zheng2014robust. For this reason, we employ the softmax model throughout this tutorial. When modeling an agent assumed to be optimal, the noise parameter $$\alpha$$ can be set to a large value.

[^softmax]: A softmax agent's choice of action is a differentiable function of their utilities. This differentiability makes possible certain techniques for inferring utilities from choices.

>**Exercise**: Monty Hall. In this exercise inspired by [ProbMods](https://probmods.org/chapters/06-inference-about-inference.html), we will approach the classical statistical puzzle from the perspective of optimal decision-making. Here is a statement of the problem:

> *Alice is on a game show and she’s given the choice of three doors. Behind one door is a car; behind the others, goats. She picks door 1. The host, Monty, knows what’s behind the doors and opens another door, say No. 3, revealing a goat. He then asks Alice if she wants to switch doors. Should she switch?*

> Use the tools introduced above to determine the answer. Here is some code to get you started:

~~~~
// Remove each element in array ys from array xs
var remove = function(xs, ys) {
  return _.without.apply(null, [xs].concat(ys));
};

var doors = [1, 2, 3];

// Monty chooses a door that is neither Alice's door
// nor the prize door
var monty = function(aliceDoor, prizeDoor) {
  return Infer({
    model() {
      var door = uniformDraw(doors);
      // ???
      return door;
    }});
};

var actions = ['switch', 'stay'];

// If Alice switches, she randomly chooses a door that is
// neither the one Monty showed nor her previous door
var transition = function(state, action) {
  if (action === 'switch') {
    return {
      prizeDoor: state.prizeDoor,
      montyDoor: state.montyDoor,
      aliceDoor: // ???
    };
  } else {
    return state;
  }
};

// Utility is high (say 10) if Alice's door matches the
// prize door, 0 otherwise.
var utility = function(state) {
  // ???
};

var sampleState = function() {
  var aliceDoor = uniformDraw(doors);
  var prizeDoor = uniformDraw(doors);
  return {
    aliceDoor,
    prizeDoor,
    montyDoor: sample(monty(aliceDoor, prizeDoor))
  }
}

var agent = function() {
  var action = uniformDraw(actions);
  var expectedUtility = function(action){
    return expectation(Infer({
      model() {
        var state = sampleState();
        return utility(transition(state, action));
      }}));
  };
  factor(expectedUtility(action));
  return { action };
};

viz(Infer({ model: agent }));
~~~~

### Moving to complex decision problems

This chapter has introduced some of the core concepts that we will need for this tutorial, including *expected utility*, *(stochastic) transition functions*, *soft conditioning* and *softmax decision making*. These concepts would also appear in standard treatments of rational planning and reinforcement learning refp:russell1995modern. The actual decision problems in this chapter are so trivial that our notation and programs are overkill.

The [next chapter](/chapters/3a-mdp.html) introduces *sequential* decisions problems. These problems are more complex and interesting, and will require the machinery we have introduced here.

### Footnotes


6c-inference-rl.md

---
layout: chapter
title: Reinforcement learning techniques
description: Max-margin and linear programming methods for IRL.
status: stub
is_section: false
hidden: true
---

- Could have appendix discussing Apprenticeship Learning ideas in Abbeel and Ng in more detail.