<!DOCTYPE HTML>
<html lang="en" class="sidebar-visible no-js light">
<head>
<!-- Book generated using mdBook -->
<meta charset="UTF-8">
<title>Layers - juice documentation</title>
<!-- Custom HTML head -->
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<meta name="description" content="Introduction book to the juice framework">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="theme-color" content="#ffffff" />
<link rel="icon" href="favicon.svg">
<link rel="shortcut icon" href="favicon.png">
<link rel="stylesheet" href="css/variables.css">
<link rel="stylesheet" href="css/general.css">
<link rel="stylesheet" href="css/chrome.css">
<link rel="stylesheet" href="css/print.css" media="print">
<!-- Fonts -->
<link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
<link rel="stylesheet" href="fonts/fonts.css">
<!-- Highlight.js Stylesheets -->
<link rel="stylesheet" href="highlight.css">
<link rel="stylesheet" href="tomorrow-night.css">
<link rel="stylesheet" href="ayu-highlight.css">
<!-- Custom theme stylesheets -->
<!-- MathJax -->
<script async type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
</head>
<body>
<!-- Provide site root to javascript -->
<script type="text/javascript">
var path_to_root = "";
var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
</script>
<!-- Work around some values being stored in localStorage wrapped in quotes -->
<script type="text/javascript">
try {
var theme = localStorage.getItem('mdbook-theme');
var sidebar = localStorage.getItem('mdbook-sidebar');
if (theme.startsWith('"') && theme.endsWith('"')) {
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
}
if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
}
} catch (e) { }
</script>
<!-- Set the theme before any content is loaded, prevents flash -->
<script type="text/javascript">
var theme;
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
if (theme === null || theme === undefined) { theme = default_theme; }
var html = document.querySelector('html');
html.classList.remove('no-js')
html.classList.remove('light')
html.classList.add(theme);
html.classList.add('js');
</script>
<!-- Hide / unhide sidebar before it is displayed -->
<script type="text/javascript">
var html = document.querySelector('html');
var sidebar = 'hidden';
if (document.body.clientWidth >= 1080) {
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
sidebar = sidebar || 'visible';
}
html.classList.remove('sidebar-visible');
html.classList.add("sidebar-" + sidebar);
</script>
<nav id="sidebar" class="sidebar" aria-label="Table of contents">
<div class="sidebar-scrollbox">
<ol class="chapter"><li class="chapter-item expanded "><a href="juice.html"><strong aria-hidden="true">1.</strong> Juice</a></li><li class="chapter-item expanded "><a href="layers.html" class="active"><strong aria-hidden="true">2.</strong> Layers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="layer-lifecycle.html"><strong aria-hidden="true">2.1.</strong> Layer Lifecycle</a></li><li class="chapter-item expanded "><a href="building-networks.html"><strong aria-hidden="true">2.2.</strong> Create a Network</a></li><li class="chapter-item expanded "><a href="create-new-layer.html"><strong aria-hidden="true">2.3.</strong> Create a new Layer</a></li></ol></li><li class="chapter-item expanded "><a href="solvers.html"><strong aria-hidden="true">3.</strong> Solvers</a></li><li><ol class="section"><li class="chapter-item expanded "><a href="optimize-layers.html"><strong aria-hidden="true">3.1.</strong> Optimize Layers</a></li><li class="chapter-item expanded "><a href="multi-device-optimization.html"><strong aria-hidden="true">3.2.</strong> Multi-Device Optimization</a></li><li class="chapter-item expanded "><a href="distributed-optimization.html"><strong aria-hidden="true">3.3.</strong> Distributed Optimization</a></li></ol></li><li class="chapter-item expanded "><a href="backend.html"><strong aria-hidden="true">4.</strong> Backend</a></li><li class="chapter-item expanded "><a href="deep-learning-glossary.html"><strong aria-hidden="true">5.</strong> Glossary</a></li><li class="spacer"></li><li class="chapter-item expanded "><a href="api-docs.html"><strong aria-hidden="true">6.</strong> Rust API Documentation</a></li></ol>
</div>
<div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
</nav>
<div id="page-wrapper" class="page-wrapper">
<div class="page">
<div id="menu-bar-hover-placeholder"></div>
<div id="menu-bar" class="menu-bar sticky bordered">
<div class="left-buttons">
<button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
<i class="fa fa-bars"></i>
</button>
<button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
<i class="fa fa-paint-brush"></i>
</button>
<ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
<li role="none"><button role="menuitem" class="theme" id="light">Light (default)</button></li>
<li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
<li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
<li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
<li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
</ul>
<button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
<i class="fa fa-search"></i>
</button>
</div>
<h1 class="menu-title">juice documentation</h1>
<div class="right-buttons">
<a href="print.html" title="Print this book" aria-label="Print this book">
<i id="print-button" class="fa fa-print"></i>
</a>
</div>
</div>
<div id="search-wrapper" class="hidden">
<form id="searchbar-outer" class="searchbar-outer">
<input type="search" name="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
</form>
<div id="searchresults-outer" class="searchresults-outer hidden">
<div id="searchresults-header" class="searchresults-header"></div>
<ul id="searchresults">
</ul>
</div>
</div>
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
<script type="text/javascript">
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
});
</script>
<div id="content" class="content">
<main>
<h1 id="layers"><a class="header" href="#layers">Layers</a></h1>
<h3 id="what-is-a-layer"><a class="header" href="#what-is-a-layer">What is a Layer?</a></h3>
<p><a href="./deep-learning-glossary.html#Layer">Layers</a> are the only building
blocks in Juice. As we will see later on, everything is a layer. Even when
we construct <a href="./deep-learning-glossary.html#Network">networks</a>, we are still just
working with layers composed of smaller layers. This makes the API clean and expressive.</p>
<p>A layer is like a function: given an input it computes an output.
It could be some mathematical expression, like Sigmoid, ReLU, or a non-mathematical instruction,
like querying data from a database, logging data, or anything in between.
In Juice, layers describe not only the interior 'hidden layers' but also the input and
output layer.</p>
<p>Layers in Juice are only slightly opinionated, they need to take
an input and produce an output. This is required in order to successfully stack
layers on top of each other to build a network. Other than that, a
layer in Juice can implement any behaviour.</p>
<p>Layers are constructed via the <a href="https://github.com/spearow/juice/blob/master/juice/src/layer.rs"><code>LayerConfig</code>
(/src/layer.rs)</a>, which makes creating even complex networks easy
and manageable.</p>
<pre><pre class="playground"><code class="language-rust">
<span class="boring">#![allow(unused)]
</span><span class="boring">fn main() {
</span>// construct the config for a fully connected layer with 500 notes
let linear_1: LayerConfig = LayerConfig::new("linear1", LinearConfig { output_size: 500 })
<span class="boring">}
</span></code></pre></pre>
<p>A <code>LayerConfig</code> can be turned into an initialized, fully operable <a href="https://github.com/spearow/juice/blob/master/juice/src/layer.rs"><code>Layer</code>
(/src/layer.rs)</a> with its <code>from_config</code> method.</p>
<pre><pre class="playground"><code class="language-rust">
<span class="boring">#![allow(unused)]
</span><span class="boring">fn main() {
</span>// construct the config for a fully connected layer with 500 notes
let linear_1: LayerConfig = LayerConfig::new("linear1", LinearConfig { output_size: 500 })
let linear_network_with_one_layer: Layer = Layer::from_config(backend, &linear_1);
<span class="boring">}
</span></code></pre></pre>
<p>Hurray! We just constructed a <a href="./deep-learning-glossary.html#Network">network</a>
with one layer. (In the following chapter we will learn how to create more
powerful networks).</p>
<p>The <code>from_config</code> method initializes a <code>Layer</code>, which wraps the specific implementation (a struct that has <a href="https://github.com/spearow/juice/blob/master/juice/src/layer.rs"><code>ILayer</code>(/src/layer.rs)</a> implemented) in a worker field.
In the tiny example above, the worker field of the <code>linear_network_with_one_layer</code>
is a <a href="https://github.com/spearow/juice/blob/master/juice/src/layers/common/linear.rs"><code>Linear</code> (/src/layers/common/linear.rs)</a> because we constructed
the <code>linear_network_with_one_layer</code> from a <code>LinearConfig</code>. The worker field
introduces the specific behaviour of the layer.</p>
<p>In the following chapters we explore more about how we can construct
real-world networks, the layer lifecycle and how we can add new layers to the Juice framework.</p>
<h3 id="what-can-layers-do"><a class="header" href="#what-can-layers-do">What can Layers do?</a></h3>
<p>A layer can implement basically any behaviour: deep learning related like
convolutions or LSTM, classical machine learning related like nearest neighbors
or random forest, or utility related like logging or normalization. To make the
behaviour of a layer more explicit, Juice groups layers into one of five
categories based on their (machine learning) functionality:</p>
<ol>
<li><a href="#Activation%20Layers">Activation</a></li>
<li><a href="#Common%20Layers">Common</a></li>
<li><a href="#Loss%20Layers">Loss</a></li>
<li><a href="#Utility%20Layers">Utility</a></li>
<li><a href="#Container%20Layers">Container.</a></li>
</ol>
<p>In practice, the groups are not really relevant, it helps make the file
structure cleaner. And it simplifies the explanation of what a layer is
doing.</p>
<h4 id="activation-layers"><a class="header" href="#activation-layers">Activation Layers</a></h4>
<p>Activation layers provide element-wise operations and return an output of
the same size as the input. Activation layers can be seen as equivalent to
nonlinear <a href="https://en.wikipedia.org/wiki/Activation_function">Activation Functions</a>
and are a fundamental piece in neural networks.</p>
<p>Examples of activation layers are <code>Sigmoid</code>, <code>TanH</code> or <code>ReLU</code>. All available
activation layers can be found at
<a href="https://github.com/spearow/juice/tree/master/juice/src/layers/activation">src/layers/activation</a>.</p>
<h4 id="loss-layers"><a class="header" href="#loss-layers">Loss Layers</a></h4>
<p>Loss layers compare an output to a target value and assign a cost to minimize.
Loss layers are often the last layer in a network.</p>
<p>Examples of loss layers are <code>Hinge Loss</code>, <code>Softmax Loss</code> or <code>Negative Log Likelihood</code>. All available loss layers can be found at
<a href="https://github.com/spearow/juice/tree/master/juice/src/layers/loss">src/layers/loss</a>.</p>
<h4 id="common-layers"><a class="header" href="#common-layers">Common Layers</a></h4>
<p>Common layers can differ in their connectivity and behavior. They are typically
anything that is not an activation or loss layer.</p>
<p>Examples of common layers are <code>fully-connected</code>, <code>convolutional</code>, <code>pooling</code>, <code>LSTM</code>,
etc. All available common layers can be found at
<a href="https://github.com/spearow/juice/tree/master/juice/src/layers/common">src/layers/common</a>.</p>
<h4 id="utility-layers"><a class="header" href="#utility-layers">Utility Layers</a></h4>
<p>Utility layers introduce all kind of helpful functionality, which might not be
directly related to machine learning and neural nets. These could be operations
for normalizing, restructuring or transforming information, log and debug
behavior or data access. Utility Layers follow the general behavior of a layer
like the other types.</p>
<p>Examples of Utility layers are <code>Reshape</code>, <code>Flatten</code> or <code>Normalization</code>. All
available utility layers can be found at
<a href="https://github.com/spearow/juice/tree/master/juice/src/layers/utility">src/layers/utility</a>.</p>
<h4 id="container-layers"><a class="header" href="#container-layers">Container Layers</a></h4>
<p>Container layers take <code>LayerConfig</code>s and connect them on initialization, which
creates a "network". But as container layers are layers themselves, one can stack multiple
container layers on top of another and compose even bigger container layers.
Container layers differ in how they connect the layers that it receives.</p>
<p>Examples of container layers are <code>Sequential</code>. All available container layers
can be found at
<a href="https://github.com/spearow/juice/tree/master/juice/src/layers/container">src/layers/container</a>.</p>
<h3 id="why-layers"><a class="header" href="#why-layers">Why Layers?</a></h3>
<p>The benefit of using a layer-based design approach is that it allows for a very expressive
setup that can represent, as far as we know, any machine learning algorithm.
That makes Juice a framework, that can be used to construct practical machine
learning applications that combine different paradigms.</p>
<p>Other machine learning frameworks take a symbolic instead of a layered approach.
For Juice we decided against it, as we found it easier for developers to work with
layers than mathematical expressions. More complex algorithms like LSTMs are
also harder to replicate in a symbolic framework. We
believe that Juices layer approach strikes a great balance between
expressiveness, usability and performance.</p>
</main>
<nav class="nav-wrapper" aria-label="Page navigation">
<!-- Mobile navigation buttons -->
<a rel="prev" href="juice.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next" href="layer-lifecycle.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
<div style="clear: both"></div>
</nav>
</div>
</div>
<nav class="nav-wide-wrapper" aria-label="Page navigation">
<a rel="prev" href="juice.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
<i class="fa fa-angle-left"></i>
</a>
<a rel="next" href="layer-lifecycle.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
<i class="fa fa-angle-right"></i>
</a>
</nav>
</div>
<script type="text/javascript">
window.playground_line_numbers = true;
</script>
<script type="text/javascript">
window.playground_copyable = true;
</script>
<script src="ace.js" type="text/javascript" charset="utf-8"></script>
<script src="editor.js" type="text/javascript" charset="utf-8"></script>
<script src="mode-rust.js" type="text/javascript" charset="utf-8"></script>
<script src="theme-dawn.js" type="text/javascript" charset="utf-8"></script>
<script src="theme-tomorrow_night.js" type="text/javascript" charset="utf-8"></script>
<script src="elasticlunr.min.js" type="text/javascript" charset="utf-8"></script>
<script src="mark.min.js" type="text/javascript" charset="utf-8"></script>
<script src="searcher.js" type="text/javascript" charset="utf-8"></script>
<script src="clipboard.min.js" type="text/javascript" charset="utf-8"></script>
<script src="highlight.js" type="text/javascript" charset="utf-8"></script>
<script src="book.js" type="text/javascript" charset="utf-8"></script>
<!-- Custom JS scripts -->
</body>
</html>
