X-Git-Url: https://fleuret.org/cgi-bin/gitweb/gitweb.cgi?p=dagnn.git;a=blobdiff_plain;f=dagnn.lua;h=b82398c0de429bd19875776fa465222a84504bbe;hp=05672e9bad5997012063b7c4f76510306cb58181;hb=HEAD;hpb=34ed0d49d9b6b03811cd92c9513edf4ec5d4d2d2 diff --git a/dagnn.lua b/dagnn.lua index 05672e9..b82398c 100755 --- a/dagnn.lua +++ b/dagnn.lua @@ -1,4 +1,23 @@ +--[[ + + Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ + Written by Francois Fleuret + + This file is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License version 3 as + published by the Free Software Foundation. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public + License for more details. + + You should have received a copy of the GNU General Public License + along with this file. If not, see . + +]]-- + require 'torch' require 'nn' @@ -6,73 +25,34 @@ local DAG, parent = torch.class('nn.DAG', 'nn.Container') function DAG:__init() parent.__init(self) - -- Nodes are indexed by the module they encompass - self.node = { } + -- Nodes are indexed by the module they contain + self.node = {} end -function DAG:createNode(nnm) - if not self.node[nnm] then - self:add(nnm) -- Add it to the object as a Container - self.node[nnm] = {} - self.node[nnm].succ = {} - self.node[nnm].pred = {} - end -end - -function DAG:addEdge(nnma, nnmb) - self.sorted = nil - self:createNode(nnma) - self:createNode(nnmb) - table.insert(self.node[nnmb].pred, nnma) - table.insert(self.node[nnma].succ, nnmb) -end - --- Apply f on t recursively; use the corresponding a1 and a2 elements --- (i.e. same keys) as second and third parameters to f when --- available; return the results from f, organized in a similarly --- nested table. -function DAG:nestApply(f, t, a1, a2) +-- Apply f on t recursively; use the corresponding elements from args +-- (i.e. same keys) as second parameter to f when available; return +-- the results from f, organized in a similarly nested table. +function DAG:nestedApply(f, t, args) if torch.type(t) == 'table' then local result = {} for k, s in pairs(t) do - result[k] = self:nestApply(f, s, a1 and a1[k], a2 and a2[k]) + result[k] = self:nestedApply(f, s, args and args[k]) end return result else - return f(t, a1, a2) + return f(t, args) end end -function DAG:setInput(i) - self.sorted = nil - self.inputModules = i - self:nestApply( - function(nnm) - if #self.node[nnm].succ == 0 then - error('Input modules must have outgoing edges.') - end - if #self.node[nnm].pred > 0 then - error('Input modules cannog have incoming edges.') - end - end, - self.inputModules - ) -end - -function DAG:setOutput(o) - self.sorted = nil - self.outputModules = o - self:nestApply( - function(nnm) - if #self.node[nnm].pred == 0 then - error('Output module must have incoming edges.') - end - if #self.node[nnm].succ > 0 then - error('Output module cannot have outgoing edges.') - end - end, - self.outputModules - ) +function DAG:createNode(nnm) + if not self.node[nnm] then + self:add(nnm) -- Add it to the object as a Container + local node = {} + node.succ = {} + node.pred = {} + node.index = #self.modules + self.node[nnm] = node + end end function DAG:putInOrder() @@ -80,15 +60,16 @@ function DAG:putInOrder() return end - -- First, we sort the nodes according to the DAG order - local distance = {} - - self:nestApply(function(m) distance[m] = 1 end, self.inputModules) + self:nestedApply( + function(m) distance[m] = 1 end, + self.inputModules + ) local nc - + local nl = 0 repeat + assert(nl < #self.modules, 'Cycle detected in the graph.') nc = 0 for nnma, node in pairs(self.node) do for _, nnmb in pairs(node.succ) do @@ -98,9 +79,14 @@ function DAG:putInOrder() end end end + nl = nl + 1 until nc == 0 - self.sorted = { } + for _, nnm in pairs(self.modules) do + assert(distance[nnm], 'Some modules are not connected to inputs.') + end + + self.sorted = {} for m, d in pairs(distance) do table.insert(self.sorted, { distance = d, nnm = m }) end @@ -110,21 +96,166 @@ function DAG:putInOrder() for i, a in ipairs(self.sorted) do self.sorted[i] = a.nnm end end +-- This accumulates x in a, where they are both nested tables of +-- tensors with same structures / keys. If first is true, set a = x +-- (in which case a can be nil) otherwise a = a + x. The behavior is +-- undefined if a and x do not have the exact same structure. +function DAG:nestedAccTensor(a, x, first) + if torch.type(x) == 'table' then + local b = {} + for i in pairs(x) do + b[i] = self:nestedAccTensor(a[i], x[i], first) + end + a = b + else + if first then + if a then + a:resizeAs(x):copy(x) + else + a = x:clone() + end + else + a:add(x) + end + end + return a +end + +function DAG:updateGradOutput(node) + local gradInputSucc = node.gradInputSucc + if #gradInputSucc == 1 then + node.gradOutput = gradInputSucc[1] + elseif #gradInputSucc > 1 then + for k = 1, #gradInputSucc do + node.gradOutput = self:nestedAccTensor(node.gradOutput, gradInputSucc[k], k == 1) + end + end +end + +---------------------------------------------------------------------- + +-- Connect a sequence of modules +function DAG:connect(...) + self.sorted = nil + local prev + for _, nnm in pairs({...}) do + self:createNode(nnm) + if prev then + table.insert(self.node[nnm].pred, prev) + table.insert(self.node[prev].succ, nnm) + end + prev = nnm + end +end + +function DAG:setLabel(nnm, label) + self.node[nnm].label = label +end + +function DAG:setInput(i) + self.sorted = nil + self.inputModules = i + self:nestedApply( + function(nnm) + assert(#self.node[nnm].succ > 0, 'Input modules must have outgoing edges.') + assert(#self.node[nnm].pred == 0, 'Input modules cannot have incoming edges.') + end, + self.inputModules + ) +end + +function DAG:setOutput(o) + self.sorted = nil + self.outputModules = o + self:nestedApply( + function(nnm) + assert(#self.node[nnm].pred > 0, 'Output module must have incoming edges.') + assert(#self.node[nnm].succ == 0, 'Output module cannot have outgoing edges.') + end, + self.outputModules + ) +end + function DAG:print() self:putInOrder() for i, d in ipairs(self.sorted) do - print('#' .. i .. ' -> ' .. torch.type(d)) + local decoration = '' + if self.node[d].label then + decoration = ' [' .. self.node[d].label .. ']' + end + print('#' .. i .. ' -> ' .. torch.type(d) .. decoration) end end +---------------------------------------------------------------------- + +function DAG:saveDot(filename) + local file = (filename and io.open(filename, 'w')) or io.stdout + + local function writeNestedCluster(prefix, list, indent) + local indent = indent or '' + if torch.type(list) == 'table' then + file:write(indent .. ' subgraph cluster_' .. prefix .. ' {\n'); + for k, x in pairs(list) do + writeNestedCluster(prefix .. '_' .. k, x, ' ' .. indent) + end + file:write(indent .. ' }\n'); + else + file:write(indent .. ' ' .. self.node[list].index .. ' [color=red]\n') + end + end + + file:write('digraph {\n') + + file:write('\n') + + writeNestedCluster('input', self.inputModules) + writeNestedCluster('output', self.outputModules) + + file:write('\n') + + for nnmb, node in pairs(self.node) do + file:write( + ' ' + .. node.index + .. ' [shape=box,label=\"' .. (self.node[nnmb].label or torch.type(nnmb)) .. '\"]' + .. '\n' + ) + + for i, nnma in pairs(node.pred) do + local decoration = '' + if #node.pred > 1 then + -- decoration = ' [headlabel=\"' .. i .. '\"]' + decoration = ' [label=\"' .. i .. '\"]' + end + file:write( + ' ' + .. self.node[nnma].index + .. ' -> ' + .. self.node[nnmb].index + .. decoration + .. '\n' + ) + end + + file:write('\n') + end + + file:write('}\n') + +end + +---------------------------------------------------------------------- + function DAG:updateOutput(input) self:putInOrder() - self:nestApply( + self:nestedApply( function(nnm, i) - self.node[nnm].input = i - nnm:updateOutput(i) + local node = self.node[nnm] + node.input = i + self:rethrowErrors(nnm, node.index, 'updateOutput', i) end, self.inputModules, input @@ -132,34 +263,47 @@ function DAG:updateOutput(input) for _, nnm in ipairs(self.sorted) do local node = self.node[nnm] - if #node.pred > 0 then + local pred = node.pred + if #pred > 0 then local i - if #node.pred == 1 then - i = node.pred[1].output - elseif #node.pred > 1 then + if #pred == 1 then + i = pred[1].output + elseif #pred > 1 then i = {} - for k = 1, #node.pred do - i[k] = node.pred[k].output + for k = 1, #pred do + i[k] = pred[k].output end end node.input = i - nnm:updateOutput(i) + self:rethrowErrors(nnm, node.index, 'updateOutput', i) end end - self.output = self:nestApply(function(m) return m.output end, self.outputModules) + self.output = self:nestedApply( + function(m) return m.output end, + self.outputModules + ) return self.output end function DAG:updateGradInput(input, gradOutput) - self:putInOrder() + assert(self.sorted, 'There has been a structure change before a DAG:updateGradInput.') - self:nestApply( - function(nnm, go) nnm:updateGradInput(self.node[nnm].input, go) end, + self:nestedApply( + function(nnm, go) + local node = self.node[nnm] + node.gradOutput = go + self:rethrowErrors(nnm, node.index, 'updateGradInput', node.input, go) + end, self.outputModules, gradOutput ) + self:nestedApply( + function(nnm, i) self.node[nnm].input = i end, + self.inputModules, input + ) + for _, node in pairs(self.node) do node.gradInputSucc = {} end @@ -167,41 +311,59 @@ function DAG:updateGradInput(input, gradOutput) for k = #self.sorted, 1, -1 do local nnm = self.sorted[k] local node = self.node[nnm] - local pred, succ, gradInputSucc = node.pred, node.succ, node.gradInputSucc - - if #gradInputSucc > 0 then - -- We update nnm:gradInput - local gi - if #gradInputSucc == 1 then - gi = gradInputSucc[1] -- we avoid a clone() - elseif #gradInputSucc > 1 then - for k = 1, #gradInputSucc do - if gi then - gi:add(gradInputSucc[k]) - else - gi = gradInputSucc[k]:clone() - end - end - end - nnm:updateGradInput(node.input, gi) + local pred = node.pred + + if #node.gradInputSucc > 0 then + self:updateGradOutput(node) + self:rethrowErrors(nnm, node.index, 'updateGradInput', node.input, node.gradOutput) end -- We fill the gradInputSucc of our predecessors if #pred == 1 then table.insert(self.node[pred[1]].gradInputSucc, nnm.gradInput) elseif #pred > 1 then - if not torch.type(nnm.gradInput) == 'table' then - error('Should have a table gradInput since it has multiple predecessors') - end + assert(torch.type(nnm.gradInput) == 'table', + 'Should have a table gradInput since it has multiple predecessors.') for n = 1, #pred do - table.insert(self.node[node.pred[n]].gradInputSucc, nnm.gradInput[n]) + table.insert(self.node[pred[n]].gradInputSucc, nnm.gradInput[n]) end end end - self.gradInput = self:nestApply(function(m) return m.gradInput end, self.inputModules) + self.gradInput = self:nestedApply( + function(m) return m.gradInput end, + self.inputModules + ) return self.gradInput end -return DAG +function DAG:accGradParameters(input, gradOutput, scale) + assert(self.sorted, 'There has been a structure change before a DAG:accGradParameters.') + + self:nestedApply( + function(nnm, go) self.node[nnm].gradOutput = go end, + self.outputModules, gradOutput + ) + + self:nestedApply( + function(nnm, i) self.node[nnm].input = i end, + self.inputModules, input + ) + + for k = 1, #self.modules do + local nnm = self.modules[k] + local node = self.node[nnm] + self:rethrowErrors(nnm, k, 'accGradParameters', node.input, node.gradOutput, scale) + end +end + +function DAG:clearState() + self.sorted = nil + for _, node in pairs(self.node) do + node.input = nil + node.gradInputSucc = nil + node.gradOutput = nil + end + return parent.clearState(self) +end