"Fossies" - the Fresh Open Source Software Archive

Member "rspamd-1.7.3/contrib/torch/optim/adagrad.lua" (10 Apr 2018, 1807 Bytes) of package /linux/misc/rspamd-1.7.3.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Lua source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file.

    1 --[[ ADAGRAD implementation for SGD
    2 
    3 ARGS:
    4 - `opfunc` : a function that takes a single input (X), the point of
    5          evaluation, and returns f(X) and df/dX
    6 - `x` : the initial point
    7 - `state` : a table describing the state of the optimizer; after each
    8          call the state is modified
    9 - `state.learningRate` : learning rate
   10 - `state.paramVariance` : vector of temporal variances of parameters
   11 - `state.weightDecay` : scalar that controls weight decay
   12 RETURN:
   13 - `x` : the new x vector
   14 - `f(x)` : the function, evaluated before the update
   15 
   16 ]]
   17 function optim.adagrad(opfunc, x, config, state)
   18    -- (0) get/update state
   19    if config == nil and state == nil then
   20       print('no state table, ADAGRAD initializing')
   21    end
   22    local config = config or {}
   23    local state = state or config
   24    local lr = config.learningRate or 1e-3
   25    local lrd = config.learningRateDecay or 0
   26    local wd = config.weightDecay or 0
   27    state.evalCounter = state.evalCounter or 0
   28    local nevals = state.evalCounter
   29 
   30    -- (1) evaluate f(x) and df/dx
   31    local fx,dfdx = opfunc(x)
   32 
   33    -- (2) weight decay with a single parameter
   34    if wd ~= 0 then
   35        dfdx:add(wd, x)
   36    end
   37 
   38    -- (3) learning rate decay (annealing)
   39    local clr = lr / (1 + nevals*lrd)
   40 
   41    -- (4) parameter update with single or individual learning rates
   42    if not state.paramVariance then
   43       state.paramVariance = torch.Tensor():typeAs(x):resizeAs(dfdx):zero()
   44       state.paramStd = torch.Tensor():typeAs(x):resizeAs(dfdx)
   45    end
   46    state.paramVariance:addcmul(1,dfdx,dfdx)
   47    state.paramStd:resizeAs(state.paramVariance):copy(state.paramVariance):sqrt()
   48    x:addcdiv(-clr, dfdx,state.paramStd:add(1e-10))
   49 
   50    -- (5) update evaluation counter
   51    state.evalCounter = state.evalCounter + 1
   52 
   53    -- return x*, f(x) before optimization
   54    return x,{fx}
   55 end