<!DOCTYPE HTML>
<html lang="en" class="sidebar-visible no-js light">
<head>
<!-- Book generated using mdBook -->
<meta charset="UTF-8">
<title>Layer Lifecycle - juice documentation</title>
<!-- Custom HTML head -->
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<meta name="description" content="Introduction book to the juice framework">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="theme-color" content="#ffffff" />
<link rel="icon" href="favicon.svg">
<link rel="shortcut icon" href="favicon.png">
<link rel="stylesheet" href="css/variables.css">
<link rel="stylesheet" href="css/general.css">
<link rel="stylesheet" href="css/chrome.css">
<link rel="stylesheet" href="css/print.css" media="print">
<!-- Fonts -->
<link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
<link rel="stylesheet" href="fonts/fonts.css">
<!-- Highlight.js Stylesheets -->
<link rel="stylesheet" href="highlight.css">
<link rel="stylesheet" href="tomorrow-night.css">
<link rel="stylesheet" href="ayu-highlight.css">
<!-- Custom theme stylesheets -->
<!-- MathJax -->
<script async type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
</head>
<body>
<!-- Provide site root to javascript -->
<script type="text/javascript">
var path_to_root = "";
var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
</script>
<!-- Work around some values being stored in localStorage wrapped in quotes -->
<script type="text/javascript">
try {
var theme = localStorage.getItem('mdbook-theme');
var sidebar = localStorage.getItem('mdbook-sidebar');
if (theme.startsWith('"') && theme.endsWith('"')) {
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
}
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
}
} catch (e) { }
</script>
<!-- Set the theme before any content is loaded, prevents flash -->
<script type="text/javascript">
var theme;
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
if (theme === null || theme === undefined) { theme = default_theme; }
var html = document.querySelector('html');
html.classList.remove('no-js')
html.classList.remove('light')
html.classList.add(theme);
html.classList.add('js');
</script>
<!-- Hide / unhide sidebar before it is displayed -->
<script type="text/javascript">
var html = document.querySelector('html');
var sidebar = 'hidden';
if (document.body.clientWidth >= 1080) {
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
sidebar = sidebar || 'visible';
}
html.classList.remove('sidebar-visible');
html.classList.add("sidebar-" + sidebar);
</script>
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
<div class="sidebar-scrollbox">
<ol class="chapter"><li class="chapter-item expanded "><a href="juice.html"><strong aria-hidden="true">1.</strong> Juice</a></li><li class="chapter-item expanded "><a href="layers.html"><strong aria-hidden="true">2.</strong> Layers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="layer-lifecycle.html" class="active"><strong aria-hidden="true">2.1.</strong> Layer Lifecycle</a></li><li class="chapter-item expanded "><a href="building-networks.html"><strong aria-hidden="true">2.2.</strong> Create a Network</a></li><li class="chapter-item expanded "><a href="create-new-layer.html"><strong aria-hidden="true">2.3.</strong> Create a new Layer</a></li></ol></li><li class="chapter-item expanded "><a href="solvers.html"><strong aria-hidden="true">3.</strong> Solvers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="optimize-layers.html"><strong aria-hidden="true">3.1.</strong> Optimize Layers</a></li><li class="chapter-item expanded "><a href="multi-device-optimization.html"><strong aria-hidden="true">3.2.</strong> Multi-Device Optimization</a></li><li class="chapter-item expanded "><a href="distributed-optimization.html"><strong aria-hidden="true">3.3.</strong> Distributed Optimization</a></li></ol></li><li class="chapter-item expanded "><a href="backend.html"><strong aria-hidden="true">4.</strong> Backend</a></li><li class="chapter-item expanded "><a href="deep-learning-glossary.html"><strong aria-hidden="true">5.</strong> Glossary</a></li><li class="spacer"></li><li class="chapter-item expanded "><a href="api-docs.html"><strong aria-hidden="true">6.</strong> Rust API Documentation</a></li></ol>
</div>
<div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
</nav>
<div id="page-wrapper" class="page-wrapper">
<div class="page">
<div id="menu-bar-hover-placeholder"></div>
<div id="menu-bar" class="menu-bar sticky bordered">
<div class="left-buttons">
<button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
<i class="fa fa-bars"></i>
</button>
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
<i class="fa fa-paint-brush"></i>
</button>
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
<li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li>
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
</ul>
<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
<i class="fa fa-search"></i>
</button>
</div>
<h1 class="menu-title">juice documentation</h1>
<div class="right-buttons">
<a href="print.html" title="Print this book" aria-label="Print this book">
<i id="print-button" class="fa fa-print"></i>
</a>
</div>
</div>
<div id="search-wrapper" class="hidden">
<form id="searchbar-outer" class="searchbar-outer">
<input type="search" name="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
</form>
<div id="searchresults-outer" class="searchresults-outer hidden">
<div id="searchresults-header" class="searchresults-header"></div>
<ul id="searchresults">
</ul>
</div>
</div>
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
<script type="text/javascript">
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
});
</script>
<div id="content" class="content">
<main>
<h1 id="layer-lifecycle"><a class="header" href="#layer-lifecycle">Layer Lifecycle</a></h1>
<p>In chapter <a href="./layers.html">2. Layers</a> we saw how to
construct a simple <code>Layer</code> from a <code>LayerConfig</code>. In this chapter, we take
a closer look at what happens inside Juice when initializing a <code>Layer</code> and when running its
<code>.forward</code> and <code>.backward</code> methods. In the next chapter <a href="./building-networks.html">2.2 Create a Network</a> we
apply our knowledge to construct deep networks with the container layer.</p>
<p>The most important methods of a <code>Layer</code> are initialization (<code>::from_config</code>), <code>.forward</code> and <code>.backward</code>.
They basically describe the entire API, so let's take a closer look at what happens inside Juice when these methods are called.</p>
<h3 id="initialization"><a class="header" href="#initialization">Initialization</a></h3>
<p>A layer is constructed from a <code>LayerConfig</code> with the <code>Layer::from_config</code>
method, which returns a fully initialized <code>Layer</code>.</p>
<pre><pre class="playground"><code class="language-rust">
<span class="boring">#![allow(unused)]
</span><span class="boring">fn main() {
</span>let mut sigmoid: Layer = Layer::from_config(backend.clone(), &LayerConfig::new("sigmoid", LayerType::Sigmoid))
let mut alexnet: Layer = Layer::from_config(backend.clone(), &LayerConfig::new("alexnet", LayerType::Sequential(cfg)))
<span class="boring">}
</span></code></pre></pre>
<p>In the example above, the first layer has a Sigmoid worker
(<code>LayerType::Sigmoid</code>) and the second layer has a Sequential worker.
Although both <code>::from_config</code> methods return a <code>Layer</code>, the behavior of
that <code>Layer</code> depends on the <code>LayerConfig</code> it was constructed with. The
<code>Layer::from_config</code> internally calls the <code>worker_from_config</code> method, which
constructs the specific worker defined by the <code>LayerConfig</code>.</p>
<pre><pre class="playground"><code class="language-rust">
<span class="boring">#![allow(unused)]
</span><span class="boring">fn main() {
</span>fn worker_from_config(backend: Rc<B>, config: &LayerConfig) -> Box<ILayer<B>> {
match config.layer_type.clone() {
// more matches
LayerType::Pooling(layer_config) => Box::new(Pooling::from_config(&layer_config)),
LayerType::Sequential(layer_config) => Box::new(Sequential::from_config(backend, &layer_config)),
LayerType::Softmax => Box::new(Softmax::default()),
// more matches
}
}
<span class="boring">}
</span></code></pre></pre>
<p>The layer-specific <code>::from_config</code> (if available or needed) then takes care of
initializing the worker struct, allocating memory for weights and so on.</p>
<p>If the worker is a container layer, its <code>::from_config</code> takes
care of initializing all the <code>LayerConfig</code>s it contains (which were added via its
<code>.add_layer</code> method) and connecting them in the order they were provided.</p>
<p>Every <code>.forward</code> or <code>.backward</code> call that is made on the returned <code>Layer</code> is
run by the internal worker.</p>
<h3 id="forward"><a class="header" href="#forward">Forward</a></h3>
<p>The <code>forward</code> method of a <code>Layer</code> threads the input through the constructed
network and returns the output of the network's final layer.</p>
<p>The <code>.forward</code> method does three things:</p>
<ol>
<li>Reshape the input data if necessary</li>
<li>Sync the input/weights to the device where the computation happens. This step
removes the need for the worker layer to care about memory synchronization.</li>
<li>Call the <code>forward</code> method of the internal worker layer.</li>
</ol>
<p>If the worker layer is a container layer, the <code>.forward</code> method
takes care of calling the <code>.forward</code> methods of its managed
layers in the right order.</p>
<h3 id="backward"><a class="header" href="#backward">Backward</a></h3>
<p>The <code>.backward</code> method of a <code>Layer</code> works similarly to <code>.forward</code>, apart from
needing to reshape the input. The <code>.backward</code> method computes
the gradient with respect to the input as well as the gradient w.r.t. the parameters. However,
the method only returns the input gradient because that is all that is needed to compute the
gradient of the entire network via the chain rule.</p>
<p>If the worker layer is a container layer, the <code>.backward</code> method
takes care of calling the <code>.backward_input</code> and
<code>.backward_parameter</code> methods of its managed layers in the right order.</p>
</main>
<nav class="nav-wrapper" aria-label="Page navigation">
<!-- Mobile navigation buttons -->
<a rel="prev" href="layers.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next" href="building-networks.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
<div style="clear: both"></div>
</nav>
</div>
</div>
<nav class="nav-wide-wrapper" aria-label="Page navigation">
<a rel="prev" href="layers.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next" href="building-networks.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
</nav>
</div>
<script type="text/javascript">
window.playground_line_numbers = true;
</script>
<script type="text/javascript">
window.playground_copyable = true;
</script>
<script src="ace.js" type="text/javascript" charset="utf-8"></script>
<script src="editor.js" type="text/javascript" charset="utf-8"></script>
<script src="mode-rust.js" type="text/javascript" charset="utf-8"></script>
<script src="theme-dawn.js" type="text/javascript" charset="utf-8"></script>
<script src="theme-tomorrow_night.js" type="text/javascript" charset="utf-8"></script>
<script src="elasticlunr.min.js" type="text/javascript" charset="utf-8"></script>
<script src="mark.min.js" type="text/javascript" charset="utf-8"></script>
<script src="searcher.js" type="text/javascript" charset="utf-8"></script>
<script src="clipboard.min.js" type="text/javascript" charset="utf-8"></script>
<script src="highlight.js" type="text/javascript" charset="utf-8"></script>
<script src="book.js" type="text/javascript" charset="utf-8"></script>
<!-- Custom JS scripts -->
</body>
</html>
