File size: 14,419 Bytes
9375c9a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 |
<html><!-- Created using the cpp_pretty_printer from the dlib C++ library. See http://dlib.net for updates. --><head><title>dlib C++ Library - approximate_linear_models_abstract.h</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2015 Davis E. King ([email protected])
</font><font color='#009900'>// License: Boost Software License See LICENSE.txt for the full license.
</font><font color='#0000FF'>#undef</font> DLIB_APPROXIMATE_LINEAR_MODELS_ABSTRACT_Hh_
<font color='#0000FF'>#ifdef</font> DLIB_APPROXIMATE_LINEAR_MODELS_ABSTRACT_Hh_
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../matrix.h.html'>../matrix.h</a>"
<font color='#0000FF'>namespace</font> dlib
<b>{</b>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'>struct</font> <b><a name='example_feature_extractor'></a>example_feature_extractor</b>
<b>{</b>
<font color='#009900'>/*!
WHAT THIS OBJECT REPRESENTS
This object defines the interface a feature extractor must implement if it
is to be used with the process_sample and policy objects defined at the
bottom of this file. Moreover, it is meant to represent the core part
of a model used in a reinforcement learning algorithm.
In particular, this object models a Q(state,action) function where
Q(state,action) == dot(w, PSI(state,action))
where PSI(state,action) is a feature vector and w is a parameter
vector.
Therefore, a feature extractor defines how the PSI(x,y) feature vector is
calculated. It also defines the types used to represent the state and
action objects.
THREAD SAFETY
Instances of this object are required to be threadsafe, that is, it should
be safe for multiple threads to make concurrent calls to the member
functions of this object.
!*/</font>
<font color='#009900'>// The state and actions can be any types so long as you provide typedefs for them.
</font> <font color='#0000FF'>typedef</font> T state_type;
<font color='#0000FF'>typedef</font> U action_type;
<font color='#009900'>// We can also say that the last element in the weight vector w must be 1. This
</font> <font color='#009900'>// can be useful for including a prior into your model.
</font> <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>bool</u></font> force_last_weight_to_1 <font color='#5555FF'>=</font> <font color='#979000'>false</font>;
<b><a name='example_feature_extractor'></a>example_feature_extractor</b><font face='Lucida Console'>(</font>
<font face='Lucida Console'>)</font>;
<font color='#009900'>/*!
ensures
- this object is properly initialized.
!*/</font>
<font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> <b><a name='num_features'></a>num_features</b><font face='Lucida Console'>(</font>
<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>;
<font color='#009900'>/*!
ensures
- returns the dimensionality of the PSI() feature vector.
!*/</font>
action_type <b><a name='find_best_action'></a>find_best_action</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> state_type<font color='#5555FF'>&</font> state,
<font color='#0000FF'>const</font> matrix<font color='#5555FF'><</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>></font><font color='#5555FF'>&</font> w
<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>;
<font color='#009900'>/*!
ensures
- returns the action A that maximizes Q(state,A) = dot(w,PSI(state,A)).
That is, this function finds the best action to take in the given state
when our model is parameterized by the given weight vector w.
!*/</font>
<font color='#0000FF'><u>void</u></font> <b><a name='get_features'></a>get_features</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> state_type<font color='#5555FF'>&</font> state,
<font color='#0000FF'>const</font> action_type<font color='#5555FF'>&</font> action,
matrix<font color='#5555FF'><</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>></font><font color='#5555FF'>&</font> feats
<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>;
<font color='#009900'>/*!
ensures
- #feats.size() == num_features()
- #feats == PSI(state,action)
!*/</font>
<b>}</b>;
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'>template</font> <font color='#5555FF'><</font>
<font color='#0000FF'>typename</font> feature_extractor
<font color='#5555FF'>></font>
<font color='#0000FF'>struct</font> <b><a name='process_sample'></a>process_sample</b>
<b>{</b>
<font color='#009900'>/*!
REQUIREMENTS ON feature_extractor
feature_extractor should implement the example_feature_extractor interface
defined at the top of this file.
WHAT THIS OBJECT REPRESENTS
This object holds a training sample for a reinforcement learning algorithm.
In particular, it should be a sample from some process where the process
was in state this->state, then took this->action action which resulted in
receiving this->reward and ending up in the state this->next_state.
!*/</font>
<font color='#0000FF'>typedef</font> feature_extractor feature_extractor_type;
<font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> feature_extractor::state_type state_type;
<font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> feature_extractor::action_type action_type;
<b><a name='process_sample'></a>process_sample</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><b>{</b><b>}</b>
<b><a name='process_sample'></a>process_sample</b><font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> state_type<font color='#5555FF'>&</font> s,
<font color='#0000FF'>const</font> action_type<font color='#5555FF'>&</font> a,
<font color='#0000FF'>const</font> state_type<font color='#5555FF'>&</font> n,
<font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font><font color='#5555FF'>&</font> r
<font face='Lucida Console'>)</font> : state<font face='Lucida Console'>(</font>s<font face='Lucida Console'>)</font>, action<font face='Lucida Console'>(</font>a<font face='Lucida Console'>)</font>, next_state<font face='Lucida Console'>(</font>n<font face='Lucida Console'>)</font>, reward<font face='Lucida Console'>(</font>r<font face='Lucida Console'>)</font> <b>{</b><b>}</b>
state_type state;
action_type action;
state_type next_state;
<font color='#0000FF'><u>double</u></font> reward;
<b>}</b>;
<font color='#0000FF'>template</font> <font color='#5555FF'><</font> <font color='#0000FF'>typename</font> feature_extractor <font color='#5555FF'>></font>
<font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> process_sample<font color='#5555FF'><</font>feature_extractor<font color='#5555FF'>></font><font color='#5555FF'>&</font> item, std::ostream<font color='#5555FF'>&</font> out<font face='Lucida Console'>)</font>;
<font color='#0000FF'>template</font> <font color='#5555FF'><</font> <font color='#0000FF'>typename</font> feature_extractor <font color='#5555FF'>></font>
<font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b> <font face='Lucida Console'>(</font>process_sample<font color='#5555FF'><</font>feature_extractor<font color='#5555FF'>></font><font color='#5555FF'>&</font> item, std::istream<font color='#5555FF'>&</font> in<font face='Lucida Console'>)</font>;
<font color='#009900'>/*!
provides serialization support.
!*/</font>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'>template</font> <font color='#5555FF'><</font>
<font color='#0000FF'>typename</font> feature_extractor
<font color='#5555FF'>></font>
<font color='#0000FF'>class</font> <b><a name='policy'></a>policy</b>
<b>{</b>
<font color='#009900'>/*!
REQUIREMENTS ON feature_extractor
feature_extractor should implement the example_feature_extractor interface
defined at the top of this file.
WHAT THIS OBJECT REPRESENTS
This is a policy based on the supplied feature_extractor model. In
particular, it maps from feature_extractor::state_type to the best action
to take in that state.
!*/</font>
<font color='#0000FF'>public</font>:
<font color='#0000FF'>typedef</font> feature_extractor feature_extractor_type;
<font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> feature_extractor::state_type state_type;
<font color='#0000FF'>typedef</font> <font color='#0000FF'>typename</font> feature_extractor::action_type action_type;
<b><a name='policy'></a>policy</b> <font face='Lucida Console'>(</font>
<font face='Lucida Console'>)</font>;
<font color='#009900'>/*!
ensures
- #get_feature_extractor() == feature_extractor()
(i.e. it will have its default value)
- #get_weights().size() == #get_feature_extractor().num_features()
- #get_weights() == 0
!*/</font>
<b><a name='policy'></a>policy</b> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> matrix<font color='#5555FF'><</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>></font><font color='#5555FF'>&</font> weights,
<font color='#0000FF'>const</font> feature_extractor<font color='#5555FF'>&</font> fe
<font face='Lucida Console'>)</font>;
<font color='#009900'>/*!
requires
- fe.num_features() == weights.size()
ensures
- #get_feature_extractor() == fe
- #get_weights() == weights
!*/</font>
action_type <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
<font color='#0000FF'>const</font> state_type<font color='#5555FF'>&</font> state
<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>;
<font color='#009900'>/*!
ensures
- returns get_feature_extractor().find_best_action(state,w);
!*/</font>
<font color='#0000FF'>const</font> feature_extractor<font color='#5555FF'>&</font> <b><a name='get_feature_extractor'></a>get_feature_extractor</b> <font face='Lucida Console'>(</font>
<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>;
<font color='#009900'>/*!
ensures
- returns the feature extractor used by this object
!*/</font>
<font color='#0000FF'>const</font> matrix<font color='#5555FF'><</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>></font><font color='#5555FF'>&</font> <b><a name='get_weights'></a>get_weights</b> <font face='Lucida Console'>(</font>
<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>;
<font color='#009900'>/*!
ensures
- returns the parameter vector (w) associated with this object. The length
of the vector is get_feature_extractor().num_features().
!*/</font>
<b>}</b>;
<font color='#0000FF'>template</font> <font color='#5555FF'><</font> <font color='#0000FF'>typename</font> feature_extractor <font color='#5555FF'>></font>
<font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> policy<font color='#5555FF'><</font>feature_extractor<font color='#5555FF'>></font><font color='#5555FF'>&</font> item, std::ostream<font color='#5555FF'>&</font> out<font face='Lucida Console'>)</font>;
<font color='#0000FF'>template</font> <font color='#5555FF'><</font> <font color='#0000FF'>typename</font> feature_extractor <font color='#5555FF'>></font>
<font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>policy<font color='#5555FF'><</font>feature_extractor<font color='#5555FF'>></font><font color='#5555FF'>&</font> item, std::istream<font color='#5555FF'>&</font> in<font face='Lucida Console'>)</font>;
<font color='#009900'>/*!
provides serialization support.
!*/</font>
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_APPROXIMATE_LINEAR_MODELS_ABSTRACT_Hh_
</font>
</pre></body></html> |