Fix backprop & LSTM weirdness

CHANGES.md

@@ -1,3 +1,16 @@
+- More backprop fixes..
+
+  *Michael Campbell*
+
+- Give LSTM input neurons linear activation.
+
+  *Michael Campbell*
+
+- For context neurons connected to a product neuron, it's initial value should
+  be 1.
+
+  *Michael Campbell*
+
 - So. Many. Bugs. Turns out the product neuron stuff was still broken and
   network evaluation wasn't behaving correctly with context neurons (recurrent
   connections). Also an error in timestep handling during backprop, and just

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rann (0.2.7)
+    rann (0.2.8)
       parallel (~> 1.12, >= 1.12.0)
       ruby-graphviz (~> 1.2, >= 1.2.3)
 

lib/rann/backprop.rb

@@ -114,11 +114,16 @@ def self.run_single network, inputs, targets
 
       # backward pass with unravelling for recurrent networks
       node_deltas = Hash.new{ |h, k| h[k] = {} }
-      gradients = Hash.new 0.to_d
-
       initial_timestep = inputs.size - 1
       neuron_stack = network.output_neurons.map{ |n| [n, initial_timestep] }
-      skipped = []
+      # initialize network end-point node_deltas in all timesteps with zero
+      network.neurons_with_no_outgoing_connections.each do |n|
+        (0...(inputs.size - 1)).each do |i|
+          node_deltas[i][n.id] = 0.to_d
+          neuron_stack << [n, i]
+        end
+      end
+      gradients = Hash.new 0.to_d
 
       while current = neuron_stack.shift
         neuron, timestep = current
@@ -137,20 +142,9 @@ def self.run_single network, inputs, targets
 
               if out_timestep > initial_timestep
                 m
+              elsif !output_node_delta
+                break
               else
-                # complicated network case, see NOTES.md
-                # can't find node delta, re-enqueue at back of queue and record
-                # the skip.
-                if !output_node_delta
-                  if skipped.size == neuron_stack.size + 1
-                    output_node_delta = 0.to_d
-                  else
-                    neuron_stack.push current
-                    skipped << current
-                    break
-                  end
-                end
-
                 # connection delta is the output neuron delta multiplied by the
                 # connection's weight
                 connection_delta =
@@ -172,8 +166,7 @@ def self.run_single network, inputs, targets
         end
 
         from_here = bptt_connecting_to neuron, network, timestep
-        neuron_stack.push *from_here
-        skipped.clear
+        neuron_stack |= from_here
 
         node_delta =
           ACTIVATION_DERIVATIVES[neuron.activation_function]
@@ -215,7 +208,7 @@ def save filepath = nil
       end
     end
 
-    def restore filepath
+    def restore filepath = nil
       unless filepath
         filepath = Dir['*'].select{ |f| f =~ /rann_savepoint_.*/ }.sort.last
 

lib/rann/lstm.rb

@@ -19,7 +19,7 @@ def initialize name, size
 
     def init
       @size.times do |j|
-        input = RANN::Neuron.new("LSTM #{name} Input #{j}", 0, :standard).tap{ |n| @network.add n }
+        input = RANN::Neuron.new("LSTM #{name} Input #{j}", 0, :standard, :linear).tap{ |n| @network.add n }
         @inputs << input
 
         f = RANN::Neuron.new("LSTM #{name} F #{j}", 3, :standard, :sig).tap{ |n| @network.add n }
@@ -36,7 +36,11 @@ def init
         memory_tanh = RANN::Neuron.new("LSTM #{name} Mem Tanh #{j}", 1, :standard, :tanh).tap{ |n| @network.add n }
         memory_o_product = RANN::ProductNeuron.new("LSTM #{name} Mem/Hidden 4 Product #{j}", 2, :standard, :linear).tap{ |n| @network.add n }
         @outputs << memory_o_product
-        memory_context = RANN::Neuron.new("LSTM #{name} Mem Context #{j}", 1, :context).tap{ |n| @network.add n }
+        memory_context =
+          RANN::Neuron.new("LSTM #{name} Mem Context #{j}", 1, :context).tap do |n|
+            @network.add n
+            n.value = 1.to_d # connecting to a product neuron
+          end
         output_context = RANN::Neuron.new("LSTM #{name} Output Context #{j}", 1, :context).tap{ |n| @network.add n }
 
         @network.add RANN::Connection.new input, f

lib/rann/network.rb

@@ -186,5 +186,13 @@ def recalculate_neuron_connection_counts!
 
       true
     end
+
+    def neurons_with_no_outgoing_connections
+      return @no_outgoing if defined? @no_outgoing
+
+      neurons.select do |n|
+        connections_from(n).none?
+      end
+    end
   end
 end

lib/rann/neuron.rb

@@ -74,7 +74,7 @@ def set_default_value!
     end
 
     def initial_activation_function
-      if standard? || context?
+      if standard?
         :relu
       else
         :linear

lib/rann/version.rb

@@ -1,3 +1,3 @@
 module RANN
-  VERSION = "0.2.7"
+  VERSION = "0.2.8"
 end