%% Version 3/21/02

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Kluwer Proceedings Sample, ProcSamp.tex
%%
%% Kluwer Academic Press
%%
%% Prepared by Amy Hendrickson, TeXnology Inc., July 1999.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%%%%
%% LaTeX2e 
%% Uncomment documentclass, 
\documentclass{kapproc} % Computer Modern font calls

%% and, optionally, one or more 
%%   of the \usepackage commands below:

%%%%%
%% If you use a font encoding package, please enter it here, i.e.,
%  \usepackage{T1enc}

%%%%%
%  If you have MathTimes and MathTimesPlus fonts, you
%  may uncomment the line below and use them, but you are
%  not obligated to do so, and most authors do not have
%  these fonts. (You may need to edit m-times.sty to make the
%  font names match those on your system)

%  You must have the MathTimes fonts for this to work. They may be
%  purchased from the Y&Y company, http://www.YandY.com.

% \usepackage[mtbold,noTS1]{m-times}

%%%%%
% PostScript font calls
%
% If you use the procps PS font file, you may need to edit it
% to make sure the font names match those on your system. See
% the top of the procps.sty file for more info.

\usepackage{procps} 

%%%%%
% Style for inserting .eps files and rotating illustrations or tables

% possible options for graphicx:
% [dvips], [xdvi], [dvipdf], [dvipsone], [dviwindo], [emtex], [dviwin],
% [pctexps],  [pctexwin],  [pctexhp],  [pctex32], [truetex], [tcidvi],
% [oztex], [textures]

%\usepackage[dvips]{graphicx}

\usepackage{graphicx}
\usepackage{epstopdf}
%% I added this to solve problems that figures are not shown. Mar 21st, 2012.

%%%%%%%%%%%%%%%%%%%%%
%% LaTeX209, 
%  Uncomment only one below, comment out similar commands above
%  \documentstyle{kapproc} % Computer Modern fonts
%  \documentstyle[procps]{kapproc} %For PostScript fonts
%  (The m-times.sty works only with LaTeX2e)

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Commands You Can Set or Change to Customize Your Book Format: ===>>>

% Running heads:
% ==============

%  Uncomment to make chapter title on left hand page
%  and section title on right hand page
%  \chapsectrunningheads


% Section heads:
% ==============

%%%
% \chaptersection % will use chapter.section form for section heads.

%%%
% Uncomment to make section heads appear in
%                    both upper and lower case.
\upperandlowercase

% \useuppercase % Uncomment to make section and subsection heads 
                %  appear in uppercase.

%%%
% How many levels of section head would you like numbered?
% 0= no section numbers, 1= section, 2= subsection, 3= subsubsection
\setcounter{secnumdepth}{1}

% Table of Contents:
% ==================
% How many levels of section head would you like to appear in the
%  Table of Contents?
%  0= chapter titles, 1= section titles, 2= subsection titles, 
%  3= subsubsection titles.

\setcounter{tocdepth}{1}

% Equation numbering:
% ===================

%%%
% \nochapequationnumber % will result in equation numbers that are (1)

%%%
% \sectionequationnumber % will result in equation numbers that are (1.1)
                         % and renumber for each section

% Default for kapproc is (equation number)

% Theorem numbering:
% ==================
% \nochaptheoremnumber % will make the theorem type environments number
       % only with the theorem number. 
       % Default is only theorem number for kapproc.

% Footnotes/Endnotes:
% ===================

% Default is endnotes that appear at the end of the chapter, above
% the references, or whereever \notes is written.

%%%
% To change footnotes to appear at bottom of page uncomment:
% \let\footnote\savefootnote

%%%
% Uncomment if you want footnotetext to appear at the bottom of the page:
%\let\footnotetext\savefootnotetext

%%%
% Uncomment if you want a ruled line above the footnote.
%\let\footnoterule\savefootnoterule

% Bibliography Style Settings:
% ============================
% Choose either kluwerbib or normallatexbib:

%%%
\kluwerbib % will produce this kind of bibliography entry:

%  Anderson, Terry L.,...
%    continuing bib entry here

%  \cite{xxx} will print without brackets around the citation.
% \bibliographystyle{kapalike} % should be used when you use \verb+\kluwerbib+.

%%%
%\normallatexbib %will produce bibliography entries as shown in the
                % LaTeX book

% [1] Anderson, Terry L.,
%     continuing bib entry

% \cite{xxx} will print with square brackets around the citation, i.e., [1].

% Any \verb+\bibliographystyle{}+ may be used with \verb+\normallatexbib+, but
% you should check with your editor to find the style preferred for
% your book.

% Change Brackets around Citation:
% ================================

%% Default with \kluwerbib is no brackets around citation. 
%% Default with \normallatexbib is square brackets around citation. 

% For parens around citation uncomment these:

%\let\lcitebracket(
%\let\rcitebracket)

% For square brackets around citation uncomment these:

%\let\lcitebracket[
%\let\rcitebracket]

% Draft Line:
% ===========
%  Optional, uncomment to make current time and `draft' appear at
%  bottom of page.

% \draft

%%%% <<== End Formatting Commands You Can Set or Change %%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




\begin{document}

\input{./cover.tex}


%------------ article title  ------------------->>

% If you use \\'s , please supply an alternate version of the title
% in square brackets, i.e., 
%\articletitle[Communism, Sparta, and Plato]
%{COMMUNISM, SPARTA,\\ and PLATO}

\articletitle[Is Bootstrapping Worth Implementing?]{Intelligent Inventory Control: \\Is Bootstrapping Worth Implementing?}

%% optional, to supply a shorter version of the title for the running head:
%%\chaptitlerunninghead{}

%\author{Tatpong Katanyukul\\
%Edwin K. P. Chong\\
%and William S. Duff}

%% multiple authors may be separated with \\
%% \author{Samuel Bostaph\\
%% and Gregor Kariotis}

%------ author/affiliation choices -------------->>

%% Single author

% \author{}
% \affil{}
% \email{}

%% Multiple authors, single affiliation

% \author{Samuel Bostaph}
% \author{George Lewis}
% \and   % <=== Type in \and before the last author so that `and' will
% \author{Cleon Jones}    % print between the last two authors
                           % in the table of contents.

% \affil{}

%% Multiple authors, multiple affiliations

% First method:
--------------

%\author{Tatpong Katanyukul}
%\affil{Faculty of Engineering, Khon Kaen University, Thailand}

%\author{Edwin K. P. Chong}
%\affil{College of Engineering, Colorado State University, USA}

%\author{William S. Duff}
%\affil{College of Engineering, Colorado State University, USA}

% Second method:
--------------

\author{Tatpong Katanyukul\altaffilmark{1}, Edwin K. P. Chong\altaffilmark{2}, 
William S. Duff\altaffilmark{3}}

\altaffiltext{1}{Department of Computer Engineering, Faculty of Engineering, Khon Kaen University, Thailand}
%\altaffiltext{2}{, Faculty of Engineering, Khon Kaen University, Thailand}
\altaffiltext{2}{Department of Electrical and Computer Engineering, College of Engineering, Colorado State University, USA}
\altaffiltext{3}{Department of Mechanical Engineering, College of Engineering, Colorado State University, USA}
%\altaffiltext{3}{Affiliation}

% Third method:
--------------

%\author{author\altaffilmark{1}, author\altaffilmark{2}, 
%         author\altaffilmark{1,3}}

%\affil{\altaffilmark{1}First affiliation, \ 
%\altaffilmark{2}Second affiliation, \
%\altaffilmark{3}Second affiliation}

%------ prologue, abstract, keywords ----------->>
% optional prologue
%\prologue{<text>}{<author, year>}


% optional abstract
\begin{abstract}
%
%This study investigates the application to an inventory problem of learning-based Approximate Dynamic Programming (ADP) approaches with and without bootstrapping techniques.
The common belief is that using Reinforcement Learning methods (RL) with bootstrapping gives better results than without. 
However, inclusion of bootstrapping increases the complexity of the RL implementation and requires significant effort. 
This study investigates whether inclusion of bootstrapping is worth the effort when applying RL to inventory problems.
Specifically, we investigate bootstrapping of the temporal difference learning method by using eligibility trace.
In addition, we develop a new bootstrapping extension to the Residual Gradient method to supplement our investigation.
The results show questionable benefit of bootstrapping when applied to inventory problems.
Significance tests could not confirm that bootstrapping had statistically significantly reduced costs of inventory controlled by a RL agent.
Our empirical results are based on a variety of problem settings, including demand correlations, demand variances, and cost structures.
%
\end{abstract}

% optional keywords
\begin{keywords}
approximate dynamic programming, inventory control, reinforcement learning, bootstrapping, eligibility trace, intelligent agent
\end{keywords}

%------------ body of article ------------------->>

%% main text
\section{Introduction}
\label{sec: introduction}

Inventory management is one of the major business activities. 
A well-managed inventory can help a business stay competitive by keeping its cash flow at a controllable level. 
%
A stochastic multiperiod inventory problem---one of the most common inventory problems---can be modeled as a Markov Decision Problem (MDP). 
%
%
%Classical approaches to the stochastic multiperiod inventory problem, one of the most common inventory problems, include analytical approaches and dynamic programming (DP).
%Analytical solution approaches are often highly problem specific and time consuming, and require specialized analytical skills and effort, as discussed in \cite{VanRoyEtAl1997a, KleinauThonemann2004a}, and \cite{JiangSheng2009a}.
%Exact dynamic programming (EDP)\footnote{
%\cite{Werbos2004a} uses the term exact dynamic programming to distinguish it from approximate dynamic programming.
%} typically requires transition probabilities, which are hard to obtain in practical inventory problems.
%EDP also suffers from the ``curse of dimensionality,'' as discussed in \cite{Bellman1957a}.
%That is, computational effort grows exponentially with problem scale.
%
Approximate Dynamic Programming (ADP) is a method to solve practical Markov decision problems.
%It does not rely on an analytical treatment of the problem or hard-to-obtain information, e.g., transition probabilities.
%Therefore ADP has received extensive attention in many control applications, as discussed by \cite{Werbos2004a}. 
%ADP uses an approximation technique, mostly either learning or simulation scheme, to overcome shortcomings of classical approaches, analytical solution and exact dynamic programming.
%
%ADP uses an approximation technique, which mostly is attained by either a learning scheme or a simulation scheme.
%A learning scheme uses a learning technique, e.g., temporal difference learning, to approximate state-action costs.
%A simulation-based ADP method uses simulation, e.g., Monte Carlo simulation, to approximate state-action costs.
Most previous studies on ADP applied to inventory management focused on learning scheme, which is also known as Reinforcement Learning (RL). %; see, e.g., \cite{VanRoyEtAl1997a}, \cite{GiannoccaroPontrandolfo2002a}, \cite{ShervaisEtAl2003a}, \cite{KimEtAl2005a}, \cite{KwonEtAl2008a}, \cite{KimEtAl2008a}, \cite{ChaharsooghiEtAl2008a}, \cite{JiangSheng2009a}, and \cite{KatanyukulEtAL2011a}.
%\cite{ChoiEtAl2006a} investigated simulation-based ADP.
%Nonetheless, these two schemes are not mutually exclusive.
%For example, \cite{KimEtAl2008a} used simulation %
%to evaluate consequences of actions not taken for accelerating a learning process in their learning-based ADP.
%
Due to its effectiveness and its link to mammal learning processes, temporal difference learning, or sometimes called ``one-step temporal-difference learning'', TD(0), is one of the most widely studied RL methods.
%
%TD(0) has been reported to be viable for general Markov decision problems.
%TD(0) uses a gradient descent method to minimize a state value approximation error by differentiation and then approximating the state value with a single-period sample.
%This effectively minimizes the error between approximations of the state value with and without a single-period sample.
%The approximation of the state value with a single-period sample can be viewed as if it is an approximation after the sample is obtained.
%On the other hand, the approximation of the state value without a single-period sample can be viewed as if it is an approximation before the sample is obtained.
%Therefore, TD(0) can be viewed as a method to minimize the error between approximations of the state value before and after a new sample is realized, and this error is often called the ``temporal difference error.''

%The TD(0) method uses the newly obtained sample to correct only the state value approximation of the previous state.
%The other prior states that led to the previous state do not get credit directly.
%The other prior states get the credit indirectly over time when later they are revisited: their values are updated and the new state values of their following state are used in the updates.
%
%To make their credit more explicit, the temporal difference learning is used with the eligibility trace \cite{Klopf1972a}.
Eligibility trace has been used to bootstrap the learning process of TD(0).
%
%This kind of mechanism is suitable for a Markov decision problem where the effect of the decision in the current period could not be observed clearly until later periods.
%In other words, using the temporal difference learning with the eligibility trace is suitable when the single period cost is delayed. 
The integration of the eligibility trace into the temporal difference learning leads to the TD($\lambda$) method.
%The TD($\lambda$) method has been widely studied, such as \cite{Tesauro1994a} and \cite{GellySilver2007a}.
% where else I have seen articles using TD(lambda)?
The success of TD($\lambda$) in many applications has lead to a general belief (see, e.g., \cite{PrestwichEtAl2008a}) that using TD($\lambda$) with $\lambda$ between 0 and 1 will give better results than TD(0).
However, the effort required to implement TD($\lambda$) is considerably higher than to implement TD(0).
The value and benefit of using TD($\lambda$) compared to TD(0) have never been investigated especially for inventory management.

%

Our study investigates the application of Sarsa($\lambda$), a widely studied implementation of the TD($\lambda$) method.
To supplement the investigation, our study develops the Direct Credit Back method (DCB)---the bootstrapped version of the Residual Gradient method (RG).
Finally, we evaluate both bootstrapping methods---Sarsa($\lambda$) and DCB---and confer to their non-bootstrapped counterparts, Sarsa and RG.
%The results revealclarify the value and benefit of bootstrapping and provide necessary information allowing sound decision on addition of bootstrapping.
The underlying methods are evaluated with various structures of inventory problems.
%We provide an empirical comparison among Sarsa, Sarsa($\lambda$), Residual Gradient, and Direct Credit Back methods.
Two other inventory control methods---Look-Ahead and Rollout---are also included in our study.
%The look-ahead method, sometimes called a rolling horizon policy, is a method commonly used in evaluation of ADP methods as commented by \cite{Powell2009a}.
%The Rollout method \cite{BertsekasTsitsiklis1996a} has been shown to be an effective inventory control method by \cite{Ours2011a}.

%Sarsa and Sarsa($\lambda$) are implementations of TD(0) and TD($\lambda$), respectively.
The findings here provide a practical approach to apply an ADP method for an inventory problem.
% and an insight relations of learning mechanism and control performance.
%Furthermore, 
The results reveal questionable benefits of bootstrapping. % for a practical inventory problem.
The understanding exposed here will help promote efficient inventory management and aid in the transfer of intelligent system research into practice.

% How paper is organized.
%This section has provided an overview of our study.
%Section \ref{sec: literature review} provides a review of related literature.
%Section \ref{sec: background} establishes the general background of Sarsa($\lambda$) and the Residual Gradient method. 
%Section \ref{sec: Direct Credit Back} explains the Direct Credit Back method.
%Section \ref{sec: Methodology} explains how this research is conducted, the problem under investigation, how each controller is set up, and how experiments are set up.
%Section \ref{sec: experiment results} presents the experimental results.
%The last section presents conclusions.

\section{Literature review}
\label{sec: literature review}

ADP has been introduced recently into inventory management research. % by \cite{VanRoyEtAl1997a}, \cite{GodfreyPowell2001a}, \cite{PontrandolfoEtAl2002a}, \cite{GiannoccaroPontrandolfo2002a}, \cite{ShervaisEtAl2003a}, \cite{KimEtAl2005a}, \cite{ChoiEtAl2006a}, \cite{TopalogluKunnumkal2006a}, \cite{IidaZipkin2006a}, \cite{ChaharsooghiEtAl2008a}, \cite{KimEtAl2008a}, \cite{KwonEtAl2008a}, \cite{PrestwichEtAl2008a}, \cite{JiangSheng2009a}, and \cite{KatanyukulEtAL2011a}.
%
%\cite{VanRoyEtAl1997a} used TD(0) with a linear combination of features, and a Multilayer Perceptron Neural Network (MLP), and showed the viability of TD(0) for inventory control problems.
%However, setting up a linear combination of features and MLP requires a high level of expertise in both application and techniques.
%\cite{ChaharsooghiEtAl2008a} used Q-learning with an aggregation
%(an enhanced version of a look-up table).
%Q-learning, introduced by \cite{Watkins1989a}, is an algorithm based on TD(0).
%An aggregation is a look-up table with a group of indices, instead of a single index.
%Any indexing value falling within the same index group will be linked to the same entry.
%For a given problem, an aggregation needs a smaller size table than a regular look-up table.
%Setting up an aggregation is generally much simpler than setting up a linear combination of features or MLP.
%
\cite{KimEtAl2005a}, \cite{KimEtAl2008a}, \cite{KwonEtAl2008a}, and \cite{JiangSheng2009a} implemented TD(0) with a look-up table as a cost-to-go approximation---a crucial component of most RL methods.
A look-up table is simple to implement and works well with TD(0).
However, a look-up table suffers from a scalability issue: its memory requirement grows exponentially as the dimension of the problem space grows.

Some studies, e.g., \cite{VanRoyEtAl1997a} and \cite{ShervaisEtAl2003a}, apply TD(0) with nonlinear function approximation, such as artificial neural network, to mitigate a scalability issue.
%
However, applying TD(0) with nonlinear function approximation requires a high level of expertise in both application and techniques and it might result in divergence leading to instability of the control.
\cite{Baird1995a} proposed Residual Gradient method (RG) to be used with nonlinear function approximation.
However, RG is reported to underperform TD(0) in overall.

In addition to extend RL method with scalable function approximation, bootstrapping is used to speed up the learning process of TD(0) and leads to a more general method, denoted TD($\lambda$).
Successes of applying TD($\lambda$)---the most famous work is \cite{Tesauro1994a}---lead to common belief of the benefit of bootstrapping.
However, due to complexity of implementation and extra computational costs, there is only work of \cite{PrestwichEtAl2008a} related to  bootstrapping and inventory management.
%
\cite{PrestwichEtAl2008a} studied the viability of combining Sarsa($\lambda$) and Noisy Genetic Algorithm by using the Cultural Algorithm, introduced by \cite{Reynolds1994a}.
%They compared the cost performance obtained from using Sarsa($\lambda$), Noisy Genetic Algorithm, and their proposed approach.
%, which is a combination of Sarsa($\lambda$) and Noisy Genetic Algorithm by the Cultural Algorithm.
They claimed the viability of their approach for partially observable Markov decision problems.
%It should be noted that since Sarsa($\lambda$) was only used as a competing method, \cite{PrestwichEtAl2008a} used the hill-climbing method to determine the parameter values of Sarsa($\lambda$), including the value of $\lambda$.

Among previous authors applying ADP to inventory problems, many authors have used TD(0) or methods related to TD(0), but none%
%
\footnote{
\cite{PrestwichEtAl2008a} used Sarsa($\lambda$) only as a competing method with hill-climbing method to determine the parameter values of Sarsa($\lambda$), but have not investigated the effectiveness of bootstrapping.}
%
 has investigated the effectiveness of bootstrapping.
%
%\cite{SuttonBarto1998a} and 
\cite{LengEtAl2009a} also mentioned that the mechanism of Eligibility Trace in ADP with function approximation has not been sufficiently investigated.

% How I have Residual Gradient here?
%In addition to TD(0), Residual Gradient (RG) method was developed by \cite{Baird1995a} to be used with nonlinear function approximation.
%Using TD(0) with nonlinear function approximation may lead to unstable system.\footnote{We are aware of later works, such as \cite{MaeiEtAl2009a}, that improve the classical temporal difference learning on this issue.}
%However, the quality of the solution that the RG method provides may be inferior to that of TD(0), as discussed in \cite{Baird1995a} and \cite{MaeiEtAl2009a}.
Inspired by the development of eligibility trace to bootstrap TD(0), we develop an extension to RG based on bootstrapping, called ``Direct Credit Back'' (DCB).
The new method DCB provides a contribution in its own right as well as supplement the investigation of an effect of bootstrapping in applying RL to inventory management.
%The investigation of such a modification to the RG method based on bootstrapping has not been explored.


\section{Background}
\label{sec: background}

\paragraph{Sarsa} Sarsa, as discussed by \cite{SuttonBarto1998a}, uses an approximate state-action cost $Q(s_t,a_t)$, often called the Q-value, to determine an action and updates $Q(s_t,a_t)$ based on TD(0). %
The equations for Sarsa are $Q(s,a) 		\gets Q(s,a) + \beta \psi(s,a)$
where $Q(s,a)$ approximates the state-action cost of state $s$ and action $a$; %
			$c(s,a)$ is a single period cost;
			$\beta$ is a Sarsa parameter, called the learning rate; %, with the other variables as defined earlier;
      the temporal difference $\psi(s,a) = c(s,a) + \alpha Q(s',a') - Q(s,a)$
and action $a_t$  is determined by a chosen policy based on state $s_t$ and the current Q-value.

\paragraph{Eligibility Trace} TD(0) uses newly observed information, $c(s_t,a_t)$, to update a Q-value of the most recent state-action.
To utilize trajectory information, Eligibility Trace (ET) is developed.
ET\footnote{ET presented here is based on \cite{SinghSutton1996a}'s replacing trace.} updates its values, such that 
when $(s,a)$ is visited, eligibility variable $e(s,a) \gets 1$, $e(s,\hat{a}) \gets 0$ for each action $\hat{a} \neq a$, and 
$e(\hat{s},b) \gets \alpha \lambda \cdot e(\hat{s},b)$ for each state $\hat{s} \neq s$ and each action $b \in A(\hat{s})$ where $\alpha$ is a discount factor; 
			$\lambda$ is an eligibility factor; %, also known as a trace-decay parameter;
			and $A(\hat{s})$ is a feasible action set for the given state $\hat{s}$.
%
Sarsa($\lambda$) is an implementation of TD($\lambda$), whose the update equation is %becomes as shown in Equation~(\ref{eq: TD(lambda)})
%
$Q(s,a) \gets Q(s,a) + \beta \cdot \psi(s,a) \cdot e(s,a)$ for all $s$ and $a$.

\paragraph{Residual Gradient method} Sarsa was originally designed to be implemented using a look-up table and later was extended for use with an approximation function.
Although there are some success, associated risk of instability is commonly known, see \cite{Baird1995a}, \cite{BarretoAnderson2008a}, and \cite{MaeiEtAl2009a}.
%
The Residual Gradient method (RG), introduced by \cite{Baird1995a}, is designed to be used with various approximation functions, including ones belonging to a non-linear family.
%
%The Residual Gradient method is based on the temporal difference learning technique which uses information obtained at different times to update the state cost approximation.
%
RG is developed to minimize the approximation error\footnote{
To be concise, the content here is presented based on state costs.
The development based on state-action costs can be conducted in a similar manner.
} %
$\xi   (\vec{\theta}) = \sum_{t=1}^T \xi_t (\vec{\theta})$ where $\xi(\vec{\theta})$ is the total approximation error and
			$\xi_t (\vec{\theta}) = \frac{1}{2} \left( C(s_t) - Q(s_t | \vec{\theta}) \right)^2$ is the approximation error of period~$t$.
Therefore, the parameter values can be determined by gradient descend method: $\vec{\theta} \gets \vec{\theta} - \beta \cdot \left( C(s_t) - Q(s_t | \vec{\theta}) \right) \nabla_{\vec{\theta}} \left( C(s_t) - Q(s_t | \vec{\theta}) \right)$.
%
%Equation~(\ref{eq: error approximation}) shows the total mean squared error of the approximation as a function of $\vec{\theta}$.
%Parameter $\vec{\theta}$ is updated with the gradient descent method, Equation~(\ref{eq: update parameter with approx error}).
%
In the development of Sarsa, partial differentiation is applied, then the real state cost $C(s_t)$ is approximated by $c(s_t) + \alpha Q(s_{t+1})$.
% and results in Sarsa with function approximation update equation, 
%
%\begin{equation}
%   \vec{\theta} \gets \vec{\theta} + \beta \cdot \left( c(s_t) + \alpha Q(s_{t+1} | \vec{\theta}) - Q(s_t | \vec{\theta}) \right) \nabla_{\vec{\theta}}  Q(s_t | \vec{\theta})
%\label{eq: Sarsa FA update}
%\end{equation}
%where $c(s_t)$ is a period cost of state $s_t$.
%
In development of RG, the approximation is applied before differentiation.
The RG update equation is
%
$\vec{\theta} \gets \vec{\theta} + \beta \cdot  \psi(s_t) \cdot \left( \alpha \nabla_{\vec{\theta}} Q(s_{t+1} | \vec{\theta}) -  \nabla_{\vec{\theta}} Q(s_t | \vec{\theta}) \right)
$ %\label{eq: Sarsa FA update}
where $\psi(s_t) = c(s_t) + \alpha Q(s_{t+1} | \vec{\theta}) - Q(s_t | \vec{\theta})$.
%
%In a similar manner, the Residual Gradient state-action update equation is obtained as
%
%\begin{equation}
%   \vec{\theta} \gets \vec{\theta} + \beta \cdot  \psi(s_t, a_t) \cdot \left( \alpha \nabla_{\vec{\theta}} Q(s_{t+1}, a_{t+1} | \vec{\theta}) -  \nabla_{\vec{\theta}} Q(s_t, a_t | \vec{\theta}) \right)
%\label{eq: Sarsa FA update}
%\end{equation}
%where $\psi(s_t, a_t) = c(s_t, a_t) + \alpha Q(s_{t+1}, a_{t+1} | \vec{\theta}) - Q(s_t, a_t | \vec{\theta})$.

\section{Direct Credit Back}
\label{sec: Direct Credit Back}

\cite{Baird1995a} claims that RG always converges.
However, %it has been reported that the Residual Gradient method converges at a slower rate than Sarsa, whenever Sarsa converges. 
it has been criticized for delivering an inferior solution compared to TD(0) by \cite{MaeiEtAl2009a}.
To improve the solution obtained from RG, we develop a Direct Credit Back (DCB) method based on bootstrapping.
%
%When new information, such as a period cost $c(s_t)$, is obtained, the Residual Gradient method uses it to update only an approximate cost of the most recent state. 
%
The DCB method uses the newly observed datum $c(s_t)$ to update the approximate costs of the most recent state as well as other prior states.

The temporal difference error of the approximate cost of state $s_t$ is
\begin{equation}
   \psi(s_t) = c(s_t) + \alpha Q(s_{t+1} | \vec{\theta}) - Q(s_t | \vec{\theta}).
\nonumber   
\end{equation}   
%
%Similarly, the temporal difference error of the approximate cost of state $s_{t-1}$ is
%\begin{equation}
%   \psi(s_{t-1}) = c(s_{t-1}) + \alpha Q(s_t | \vec{\theta}) - Q(s_{t-1} | \vec{\theta}).
%\nonumber   
%\end{equation}   
After the period cost $c(s_t)$ is observed, $Q(s_t | \vec{\theta})$ can be approximated by $c(s_t) + \alpha Q(s_{t+1} | \vec{\theta})$.
Therefore, the temporal difference error of the approximate cost of state $s_{t-1}$ after $c(s_t)$ is observed is defined as
\begin{equation}
   \psi(s_{t-1}|c(s_t)) = c(s_{t-1}) + \alpha \left(c(s_t) + \alpha Q(s_{t+1} | \vec{\theta}) \right) - Q(s_{t-1} | \vec{\theta}).
\label{eq: TD previous state}   
\end{equation}   
Equation~(\ref{eq: TD previous state}) can be rearranged as shown in Equation~(\ref{eq: TD previous state TD term}),
\begin{equation}
   \psi(s_{t-1}|c(s_t)) = c(s_{t-1}) + \alpha \left(\psi(s_t | c(s_t)) + Q(s_t | \vec{\theta}) \right) - Q(s_{t-1} | \vec{\theta})
\label{eq: TD previous state TD term}   
\end{equation}   
where $\psi(s_t | c(s_t)) = \psi(s_t)$.

The update equations for other prior states can be obtained in a similar manner.
After $c(s_t)$ is observed, the temporal difference error of approximate cost of state $s_{t-i}$, for $i = 1,2,\ldots, t-1$, is shown in Equation~(\ref{eq: TD prior state}):
\begin{equation}
   \psi(s_{t-i}|c(s_t)) = c(s_{t-i}) + \alpha \left(\psi(s_{t-i+1} | c(s_t)) + Q(s_{t-i+1} | \vec{\theta}) \right) - Q(s_{t-i} | \vec{\theta}).
\label{eq: TD prior state}   
\end{equation}   

Minimizing the squared temporal difference errors of all prior states is equivalent to minimizing $\sum_{i=0}^{t-1} \psi^2(s_{t-i}|c(s_t))$.
The DCB update equation can be obtained by the gradient descent method. %, 
%\begin{equation}
%   \vec{\theta} \gets \vec{\theta} - \beta \sum_{i=0}^{t-1} \psi(s_{t-i}|c(s_t)) 
%      \cdot \left( \alpha^{i+1} \cdot \nabla_{\vec{\theta}} Q(s_{t+1}|\vec{\theta}) - \nabla_{\vec{\theta}} Q(s_{t-i}|\vec{\theta}) \right).
%\label{eq: DCB update}
%\end{equation}
%
The update can be truncated to only a specific number of prior approximate state costs, 
\begin{equation}
   \vec{\theta} \gets \vec{\theta} - \beta \sum_{i=0}^{\min \{t-1, N_{cb}\}} \psi(s_{t-i}|c(s_t)) 
      \cdot \left( \alpha^{i+1} \cdot \nabla_{\vec{\theta}} Q(s_{t+1}|\vec{\theta}) - \nabla_{\vec{\theta}} Q(s_{t-i}|\vec{\theta}) \right)
\label{eq: truncated DCB update}
\end{equation}
where $N_{cb} \in \{0, 1, 2, \ldots \}$ is the number of periods crediting back.
The update equation for approximate state-action costs can be obtained in a similar manner.
When the parameter $N_{cb} = 0$, the DCB method reduces to RG.

\section{Experiments}
\label{sec: Methodology}

Our study uses computer simulations to conduct numerical experiments.
The inventory problem investigated here is a periodic review single-echelon problem with nonzero leadtime and a setup cost.

The demand is modeled as AR1/GARCH(1,1).
Therefore, a state is composed of the previous demand $D_{t-1}$, the previous demand error $\epsilon_{t-1}$, the variance of the previous demand error $\sigma_{t-1}^2$, the on-site inventory $x_t$, and the in-transit inventory $\vec{B}^{(t)}$ whose length depends on a leadtime $L$.
The state space of this problem is $\{0, \mathcal{I}^+\}$ $\times$  $\Re$ $\times$  $\Re$ $\times$ $\mathcal{I}$ $\times$ $\{0, \mathcal{I}^+\}^L$ for $D_{t-1}$, $\epsilon_{t-1}$, $\sigma_{t-1}^2$, $x_t$, and $\vec{B}^{(t)}$, respectively.
An action $u_t \in \{0, \mathcal{I}^+\}$ is a replenishment order.
State transitions are specified by (1) $D_t = a_0 + a_1 \cdot D_{t-1} + \epsilon_t$, 
(2) $\epsilon_t = e_t \cdot \sigma_t$, 
(3) $\sigma_t^2 = \nu_0 + \nu_1 \cdot \epsilon_{t-1}^2 + \nu_2 \cdot \sigma_{t-1}^2$,
(4) $x_{t+1} = x_t + B_1^{(t)} - D_t$, and
(5) $\vec{B}^{(t+1)} = \left[ B_1^{(t+1)} \; \cdots \; B_{L-1}^{(t+1)} \; B_L^{(t+1)} \right]^T = 
  \left[ B_2^{(t)} \; \cdots \; B_L^{(t)} \; u_t \right]^T$
where $a_0$ and $a_1$ are AR1 model parameters;
			$\nu_0$, $\nu_1$, and $\nu_2$ are GARCH(1,1) parameters;
			$e_t$ is white noise distributed according to $N(0,1)$; and $u_t$ is the replenishment order. %, and $t = 2, 3, 4, \ldots$.

%The period cost, or transition cost, is the cost associated with the system transitioning from the current state to the next state by the chosen action.
The period cost consists of the replenishment cost and the inventory handling cost, $c_{t+1}
   = K_t \cdot \delta(u_t) + g_t \cdot u_t	+ h_t \cdot (x_{t+1})^+ + b_t \cdot (-x_{t+1})^+$
where $c_{t+1}$ is the period cost, whose value will be known at time $t+1$ (the end of period $t$); %(therefore, it is indexed with $t+1$);
			$K_t$ is the setup cost;
			$g_t$ is the unit replenishment cost;
			$h_t$ is the unit holding cost;
			$b_t$ is the unit backlogging cost;
%			$s^{(t+1)}_2$, which equals $x_{t+1}$, is a second component of state $\vec{s}_{t+1}$;
%			$a_t$, which equals $\mathring{u}_t$, is a replenishment order;
			$\delta(\cdot)$ is the step function;
			$(\cdot)^+$ is the positive function defined by $(a)^+ = a \cdot \delta(a)$;
			and other variables are as mentioned earlier.
%The first two terms in Equation~(\ref{eq: problem transition cost}) represent the replenishment cost and the last two terms represent the handling cost.

Our study investigated 13 different inventory problem structures (as shown in Table~\ref{tbl: a summary of demand and cost variables investigated}) with other parameters held fixed: AR1's $a_0$ = 2, GARCH(1,1)'s $\nu_1$ = 0.1 and $\nu_2$ = 0.8, a discount factor $\alpha$ = 1, a leadtime $L$ = 1, and the unit replenishment cost $g$ = \$100/unit.
Each experiment is initialized at $D_0$ = 50, $\epsilon_0$ = 10, $\sigma_0^2$ = 400, $x_1$ = 10, and $B^{(1)}$ = 0.
%
Each method's performance is evaluated based on its aggregate cost of Periods 13--60.
Using an aggregate cost allows ADP methods to have initial learning periods and thus provides better performance evaluation of ADP methods.
%
The experiments run each inventory controller for 50 replications of each of 60 time-unit-indeterminate periods.

%The study of problems 1 to 3 provides performance comparison across different values of $a_1$ of 0.8, 0, and $-0.8$, respectively.
%The study of problems 1, 4, and 5 provides performance comparison across different values of $\nu_0$ of 70, 40, and 10, respectively.
%The study of problems 6 to 9 provides performance comparison across different values of $K$ of 150, 100, 50, and 0, respectively.
%The study of problems 10, 9, and 11 provides performance comparison across different values of $b$ of 200, 100, and 50, respectively.
%The study of problems 11 to 13 provides performance comparison across different values of $h$ of 1, 10, and 25, respectively.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Table problem structures

\begin{table}[hbtp]
\caption{A summary of demand and cost variables investigated}
\begin{center}
{\scriptsize %\tiny
\begin{tabular}{|r|c|c|c|c|c|c|}
\hline
% Header line 1
         &  \multicolumn{2}{|c|}{demand variables} &  \multicolumn{3}{|c|}{cost variables}   \\
\cline{2-6}
  	     & correlation & noise variance offset  &  set up cost  & penalty cost & holding cost   \\
         & ($a_1$)  	  & ($\nu_0$) 						 &  ($K$: \$/transaction) 			 & ($b$: \$/item) 			  & ($h$: \$/item) \\
\hline
P1  & 0.8 & 70 & 100 & 200 & 1 \\ %& Table~\ref{tbl: summary P166} \\
P2  & 0 & 70 & 100 & 200 & 1 \\ %& Table~\ref{tbl: summary P167} \\
P3  & -0.8 & 70 & 100 & 200 & 1 \\ %& Table~\ref{tbl: summary P168} \\
P4  & 0.8 & 40 & 100 & 200 & 1 \\ %& Table~\ref{tbl: summary P169} \\
P5  & 0.8 & 10 & 100 & 200 & 1 \\ %& Table~\ref{tbl: summary P170} \\
P6  & 0.8 & 70 & 150 & 100 & 1 \\ %& Table~\ref{tbl: summary P157} \\
P7  & 0.8 & 70 & 100 & 100 & 1 \\ %& Table~\ref{tbl: summary P164} \\
P8 & 0.8 & 70 & 50 & 100 & 1 \\ %& Table~\ref{tbl: summary P163} \\
P9 & 0.8 & 70 & 0 & 100 & 1 \\ %& Table~\ref{tbl: summary P158} \\
P10 & 0.8 & 70 & 0 & 200 & 1 \\ %& Table~\ref{tbl: summary P159} \\
P11 & 0.8 & 70 & 0 & 50 & 1 \\ %& Table~\ref{tbl: summary P160} \\
P12 & 0.8 & 70 & 0 & 50 & 10 \\ %& Table~\ref{tbl: summary P161} \\
P13 & 0.8 & 70 & 0 & 50 & 25 \\ %& Table~\ref{tbl: summary P165} \\ 
%
\hline
%
\end{tabular}
} %footnotesize
\end{center}
\label{tbl: a summary of demand and cost variables investigated}
\end{table}

\paragraph{Implementation} Sarsa, Sarsa($\lambda$), RG, and DCB methods are used with the Radial Basis Function (RBF) as the approximation function.
RBF has three sets of parameters: centers, scales, and weights.
RBF centers and scales are set up as \cite{KatanyukulEtAL2011a}.
RBF weights are adjustable parameters whose values are determined by the method under investigated.
%RBF centers are at each combinations of \{0, 100, 200, 300\} $\times$ \{-300, -200, -100, ..., 300\} $\times$ \{0, 100, 200, 300, 400\} $\times$ \{0, 100, 200, 300, 400\}, corresponding to $D_{t-1}$, $x_t$, $B^{(t)}$, and $u_t$, respectively.
%RBF scales are $z_1$ = $z_2$ = $z_3$ = $z_4$ = $0.4621 \times 10^{-4}$.
% and $z_3$ = $0.0072 \times 10^{-4}$.
Variables $\epsilon_{t-1}$ and $\sigma_{t-1}^2$ are excluded from state parameters for reasons discussed in \cite{KatanyukulEtAL2011a}.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experimental results}
\label{sec: experimental results}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

We measure performance of each inventory control by aggregate costs.
%
Figures \ref{fig: significance tests P1 DEMAND} and \ref{fig: significance tests P2 DEMAND} show averages and confidence intervals of aggregate costs obtained from different methods under problem scenarios P1--P13 (Table \ref{tbl: a summary of demand and cost variables investigated}).
A `*' marks an average aggregate cost and line beside it represents a 90$\%$ confidence interval, based on t-test statistics.
On the y-axis, labels `H', 'S', 'SL', 'RG', and 'DCB', and `Roll' indicate results obtained from the 12-period Look-Ahead, Sarsa, Sarsa($\lambda$), RG, and the DCB method, respectively\footnote{Results of each method are presented with the best performing set of parameter values.}.
The Roll-out method was used with perfect system information: Rollout simulation parameter values match those of the actual problem.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% chkpp2sigtest01b.m
%
\begin{figure}
\begin{center}
\includegraphics[width=3in]{chkpp2meanCI01P1.eps}
\end{center}
\caption{Means and Confidence Intervals of aggregate costs}
\label{fig: significance tests P1 DEMAND}
\end{figure}
%
\begin{figure}
\begin{center}
\includegraphics[width=3in]{chkpp2meanCI01P2.eps}
\end{center}
\caption{Means and Confidence Intervals of aggregate costs}
\label{fig: significance tests P2 DEMAND}
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Discussion and Conclusions}
\label{sec: Discussion and Conclusions}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

The average costs obtained from RL methods with bootstrapping are lower than without in most scenarios, but P13 that the average cost obtained from DCB is higher than one from RG method.
Figures \ref{fig: relative cost difference DEMAND} and \ref{fig: relative cost difference COST} illustrate the effect of bootstrapping.
The y-axes show the percentage relative cost difference of method A to method B, denoted ``$\%$ rel.\ cost\ diff.\ '', which is $(cost_A / cost_B - 1) \times 100 \%$.
The x-axes show scenario parameters as indicated: demand correlations ($a_1$), variances ($\nu_0$), set up per unit cost ($K/g$), penalty cost per unit cost ($b/g$), and holding cost per penalty cost ($h/b$).
The plots in the first columns show $\%$ rel.\ cost\ diff.\ of each RL method, as indicated by the legends, to the 12-period Look-Ahead method (H12).
Similarly, the plots in the middle and last columns show $\%$ rel.\ cost\ diff.\ to Sarsa (S) and the Residual Gradient method (RG), respectively.

Based on percentages of relative cost difference, bootstrapping reduces average cost up to about 5$\%$, regardless of the RL method.
%
It should be noted that scenario P13 that DCB has higher average cost than RG is when the holding cost ratio ($h/b$) is relatively high.
When the holding cost is relatively high, not only that DCB underperforms RG method, all methods---Sarsa, Sarsa($\lambda$), RG, and DCB methods---underperform the 12-period Look-Ahead method (bottom left plot of Figure \ref{fig: relative cost difference COST}).
This implies that RL method does not work well under high $h/b$ ratio.
High $h/b$ ratios make inventory decisions more critical---stocking more inventory has more negative effect.
This may cause slower convergence of all RL methods.
The study of suitable ADP methods for critical decision problems deserves further investigation.

Although the reduction of average aggregate costs by using bootstrapping seem apparent in most cases, the reduction is still within ranges of variation (at 95$\%$ confidence level).
Therefore, statistically we cannot rule out the possibilities of the variation of results due to stochastic variations in the problems.
We have run significance tests to double-check, but the test results could not confirm the cost reduction of bootstrapping at 95$\%$ confidence level.
Table \ref{tbl: significance test results} shows p-values of the significance tests.
The higher p-value, i.e., closer to 1, implies that evidence against the null hypothesis is insufficient.

To conclude the point, bootstrapping has shown to be able to reduce average cost up to 5$\%$ with possibility that this reduction may be due to variation of the underlying process.
Therefore, taking into consideration the additional implementation effort and computation cost required by bootstrapping, bootstrapping in the forms investigated in this study are not recommended for inventory control applications.

It should be noted that our conclusion is drawn based on our experimental results.
Investigation of different form of bootstrapping or different problem structure, such as a problem with long leadtime, may reveal a situation where bootstrapping can work more effectively.


%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{table}[hbtp]
\caption{Significance test results}
\begin{center}
\begin{tabular}{|l|c|c|c|c|c|c|c|}
%
\hline
\multicolumn{8}{|c|}{p-values obtained from t-tests} \\
\hline
Scenario & P1 & P2 & P3 & P4 & P5 & P6 & \multicolumn{1}{|c|}{P7} \\
\hline
$H_a$: $cost_{S} \ne cost_{S(\lambda)}$ & 0.76 & 0.60 & 0.39 & 0.75 & 0.57 & 0.66 & \multicolumn{1}{|c|}{(0.45)} \\
$H_a$: $cost_{RG} \ne cost_{DCB}$ & (0.86) & 0.65 & 0.23 & 0.64 & 0.77 & (0.39) & \multicolumn{1}{|c|}{(0.97)} \\
\hline
%
\hline
\hline
Scenario & P8 & P9 & P10 & P11 & P12 & P13 & \multicolumn{1}{|c|}{ }\\
\hline
$H_a$: $cost_{S} \ne cost_{S(\lambda)}$ & 0.63 & 0.96 & 0.44 & (0.66) & 0.82 & (0.69) & \multicolumn{1}{|c|}{ } \\
$H_a$: $cost_{RG} \ne cost_{DCB}$ & (0.71) & (0.52) & (0.58) & (0.43) & 0.92 & 0.12  & \multicolumn{1}{|c|}{ } \\
\hline
%
\multicolumn{8}{l}{ \scriptsize{Remark: p-values in parentheses represent p-values obtained from Wilcoxon ranksum tests.}} \\
\multicolumn{8}{l}{\scriptsize{Wilcoxon ranksum test is used when assumption of normality is doubted.}}\\
\multicolumn{8}{l}{\scriptsize{The normal assumption is tested by Lilliefor test at $5\%$ significance level.}}\\
%\multicolumn{8}{l}{The multiple comparison is based on Tukey's honestly significant difference criterion and one-way ANOVA.}\\
%
\end{tabular}
\end{center}
\label{tbl: significance test results}
\end{table}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% pp2corvarsr04ab.m
%
\begin{figure}
\begin{center}
\includegraphics[width=3in]{pp2corvarsr04abLarge.eps}
\end{center}
\caption{Relative cost differences of ADP methods on different demand correlations (upper row) and variances (lower row)}
\label{fig: relative cost difference DEMAND}
\end{figure}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% chkP15costsr04ab.m
%
\begin{figure}
\begin{center}
\includegraphics[width=3in]{pp2costsr04abLarge.eps}
\end{center}
\caption{Relative cost differences of ADP methods on different set up costs (top row), penalty costs (middle row) and holding costs (bottom row)}
\label{fig: relative cost difference COST}
\end{figure}


%------------ end of article ------------------->>

\begin{chapthebibliography}{1}
\bibitem{ander}

\bibitem[{Baird~(1995)}]{Baird1995a}
L.~Baird. {\it Residual Algorithms: Reinforcement
  Learning with Function Approximation}, in Proceedings of
  the 12th International Conference on Machine Learning,
  Morgan Kaufmann, 1995, 30--37.

\bibitem[{Barreto and Anderson~(2008)}]{BarretoAnderson2008a}
A.~M.~S.~Barreto and C.~W.~Anderson. {\it Restricted gradient-descent algorithm for value-function approximation in reinforcement learning}. Artificial Intelligence, 172 (4--5) 2008, 454--482.

\bibitem[{Jiang and Sheng~(2009)}]{JiangSheng2009a}
C.~Jiang and Z.~Sheng. {\it Case-based reinforcement learning for dynamic inventory
  control in a multi-agent supply chain system}. Expert Systems with Applications, 36 (3) 2009, 6520--6526.

\bibitem[{Katanyukul et~al.~(2011)}]{KatanyukulEtAL2011a}
T.~Katanyukul, W.~S.~Duff, and E.~K.~P.~Chong. {\it Approximate dynamic programming for an inventory problem: Empirical comparison}, Computers \& Industrial Engineering, 60 (4) 2011, 719--743.

\bibitem[{Kim et~al.~(2005)}]{KimEtAl2005a}
C.~O. Kim, J.~Jun, J.~K. Baek, R.~L. Smith, and Y.~D. Kim. {\it Adaptive inventory control models for supply chain management}, International Journal of Advanced Manufacturing Technology, 26 (9--10) 2005, 1184--1192.

\bibitem[{Kim et~al.~(2008)}]{KimEtAl2008a}
C.~O. Kim, I.~H. Kwon, and J.~G. Baek. {\it Asynchronous action-reward
  learning for nonstationary serial supply chain inventory control}, Applied Intelligence 28 (1) (2008), 1--16.

\bibitem[{Kwon et~al.~(2008)}]{KwonEtAl2008a}
I.~H. Kwon, C.~O. Kim, J.~Jun, and J.~H. Lee. {\it Case-based myopic reinforcement learning for satisfying
  target service level in supply chain}, Expert Systems with Applications 35 (1--2) 2008, 389--397.

\bibitem[{Leng et~al.~(2009)}]{LengEtAl2009a}
J.~Leng, L.~Jain, and C.~Fyfe. {\it Experimental analysis of eligibility traces strategies in temporal difference learning}, International Journal of Knowledge Engineering and Soft Data Paradigms, 1 (1) 2009, 26--39.

\bibitem[Maei et~al.~(2009)]{MaeiEtAl2009a}
H.~R.~Maei, Cs.~Szepesvari, S.~Bhatnagar, D.~Precup, D.~Silver, and R.~S.~Sutton.
{\it Convergent Temporal-Difference Learning with Arbitrary Smooth Function Approximation}, in Advances in Neural Information Processing Systems, MIT Press, Vancouver, BC., 2009.

\bibitem[{Prestwich et~al.~(2008)}]{PrestwichEtAl2008a}
S.~D. Prestwich, S.~A. Tarim, R.~Rossi, and B.~Hnich.
{\it A Cultural Algorithm for POMDPs from Stochastic Inventory Control}, in HM '08: Proceedings of the 5th International Workshop on Hybrid Metaheuristics, Springer-Verlag, Berlin, Heidelberg, 2008, 16--28.

\bibitem[{Reynolds~(1994)}]{Reynolds1994a}
R.~G. Reynolds. {\it An Introduction to Cultural Algorithms}, in Proceedings of the 3rd Annual Conference on Evolutionary Programming, World Scientific Publishing, 1994, 131--139.

\bibitem[{Shervais et~al.~(2003)}]{ShervaisEtAl2003a}
S.~Shervais, T.~T. Shannon, and G.~G. Lendaris. {\it Intelligent Supply Chain Management Using Adaptive Critic Learning}, IEEE Transactions on Systems, Man, and Cybernetics-Part A: Systems and Humans 33 (2) 2003, 235--244.

\bibitem[{Singh and Sutton~(1996)}]{SinghSutton1996a}
S.~P. Singh and R.~S. Sutton. {\it Reinforcement Learning with Replacing Eligibility Traces}, Machine Learning 22 (1--3) 1996, 123--158.

\bibitem[{Sutton and Barto~(1998)}]{SuttonBarto1998a}
R.~S. Sutton and A.~G. Barto. {\it Reinforcement Learning}, MIT Press, 1998.

\bibitem[{Tesauro~(1994)}]{Tesauro1994a}
G.~J. Tesauro. {\it {TD}-Gammon, a self-teaching backgammon program, achieves master level play}. 
Neural Computation, 6 (2) 1994, 215--219.

%\bibitem[{Van~Roy et~al.(1997)Van~Roy, Bertsekas, Lee, and Tsitsiklis}]{VanRoyEtAl1997a}
\bibitem[{Van~Roy et~al.~(1997)}]{VanRoyEtAl1997a}
B.~Van~Roy, D.~P. Bertsekas, Y.~Lee, and J.~N. Tsitsiklis. {\it A Neuro-Dynamic Programming Approach to Retailer Inventory Management}, in Proceedings of the IEEE Conference on Decision and Control, 1997.

\end{chapthebibliography}

%\bibliographystyle{apalike}
%\chapbblname{Boots}
%\chapbibliography{guilinchapbib01}

%\bibliographystyle{kapalike}
%\begin{chapthebibliography}
%\begin{thebibliography}{17}
%\providecommand{\natexlab}[1]{#1}
%\providecommand{\url}[1]{\texttt{#1}}
%\providecommand{\urlprefix}{URL }
%\expandafter\ifx\csname urlstyle\endcsname\relax
%  \providecommand{\doi}[1]{doi:\discretionary{}{}{}#1}\else
%  \providecommand{\doi}[1]{doi:\discretionary{}{}{}\begingroup
%  \urlstyle{rm}\url{#1}\endgroup}\fi
%\providecommand{\bibinfo}[2]{#2}

%\bibitem[{Katanyukul et~al.~(2011)}]{KatanyukulEtAL2011a}
%\bibinfo{author}{T.~Katanyukul}, 
%\bibinfo{author}{W.~S.~Duff}, 
%\bibinfo{author}{E.~K.~P.~Chong}, 
%\bibinfo{title}{Approximate dynamic programming for an inventory problem: Empirical comparison},
%\bibinfo{journal}{Journal of Computers \& Industrial Engineering}, 
%  \bibinfo{volume}{60}~(\bibinfo{number}{4}) (\bibinfo{year}{2011})
%  \bibinfo{pages}{719--743}.

%\end{chapthebibliography}

\end{document}

%Other commands, and notes on usage:

-----
%Possible section head levels:
%\section{Introduction}
%\subsection{This is subsection}
%\subsubsection{This is subsubsection}
%\paragraph{This is the paragraph}

-----
Captions:
 If you use index commands within a caption, precede \inx or \inxx with
 \protect.

\begin{figure}[h]
\caption{\protect\inx{Oscillograph} for memory address access ....
memory plane.}
\end{figure}

-----
Tables:
 Remember to use \centering for a small table and to start the table
 with \hline, use \hline underneath the column headers and at the end of 
 the table. Column headers should be in italic:

\begin{table}[h]
\caption{Small Table}
\centering
\begin{tabular}{ccc}
\sphline
\it one&\it two&\it three\\
\sphline
C&D&E\\
\sphline
\end{tabular}
\end{table}

For a table that expands to the width of the page, write

\begin{table}
\begin{tabular*}{\textwidth}{@{\extracolsep{\fill}}lcc}
\hline
....
\end{tabular*}
%% Sample table notes:
\begin{tablenotes}
$^a$Refs.~19 and 20.

$^b\kappa, \lambda>1$.
\end{tablenotes}
\end{table}

-----
%% Use \printindex if you are using the LaTeX Makeindex commands:
%% \makeindex and \index{}
%\printindex

%% Use \kluwerprintindex if you are using the Kluwer indexing macros:
%%(\inx{} and \inxx{})
\kluwerprintindex

Kluwer Index commands:
\inx{term} will print `term' in text but will also send `term' and its
page number to the .inx file.

\inxx{term} will not print in text but will send term and its page number to
the .inx file.

\inxx{term,second term} will not print in text but will send `second term'
to inx file to print underneath `term' in the index.

1) Run Latex on file,
2) Run sort routine on file ie. `sort < filename.inx > filename.srt' on
your system to produce a filename.srt file
3)\printindex at end of book will input filename.srt and print index.

See documentation for more information.

-----
Algorithm.
Maintains same fonts as text (as opposed to verbatim which uses fixed
width fonts). Space at beginning of line will be maintained if you
use \ at beginning of line.

\begin{algorithm}
{\bf state\_transition algorithm} $\{$
\        for each neuron $j\in\{0,1,\ldots,M-1\}$
\        $\{$   
\            calculate the weighted sum $S_j$ using Eq. (6);
\            if ($S_j>t_j$)
\                    $\{$turn ON neuron; $Y_1=+1\}$   
\            else if ($S_j<t_j$)
\                    $\{$turn OFF neuron; $Y_1=-1\}$   
\            else
\                    $\{$no change in neuron state; $y_j$ remains %
unchanged;$\}$ .
\        $\}$   
$\}$   
\end{algorithm}

-----
Sample quote:
\begin{quote}

\end{quote}

-----
Listing samples

\begin{enumerate}
\item
This is the first item in the numbered list.

\item
This is the second item in the numbered list.
\end{enumerate}

\begin{itemize}
\item
This is the first item in the itemized list.

\item
This is the first item in the itemized list.
This is the first item in the itemized list.
This is the first item in the itemized list.
\end{itemize}

\begin{itemize}
\item[]
This is the first item in the itemized list.

\item[]
This is the first item in the itemized list.
This is the first item in the itemized list.
This is the first item in the itemized list.
\end{itemize}

----
Glossary:

\begin{glossary}
\term{xxx}Text...
\term{yyy}Text...
\end{glossary}

i.e.,
\begin{glossary}
\term{GaAs}Gallium Arsinide. For similar device sizes GaAs transistors 
have three to
five times greater transconductance than those of of silicon bipolar
and MOS transistors.

\term{VLSI}Very Large Scale Integration. Since the mid-1970's 
VLSI technology has been successfully used in many areas, but its effect on
computers of all shapes and sizes has been the most dramatic. Some of the
application areas got boosts in performance while others became
feasible.

\end{glossary}







