diff --git a/docs/references.bib b/docs/references.bib index d8c39873..b46124cd 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -189,7 +189,21 @@ @article{DeMarchi2022 @article{DeMarchi2024, title={An interior proximal gradient method for nonconvex optimization}, author={De Marchi, Alberto and Themelis, Andreas}, - journal={arXiv:2208.00799v2}, + journal={Open Journal of Mathematical Optimization}, + volume={5}, + eid = {3}, + pages = {1--22}, year={2024}, - url={https://doi.org/10.48550/arXiv.2208.00799} + url={https://doi.org/10.5802/ojmo.30} +} + +@article{Latafat2025, + title={Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient}, + author={Latafat, Puya and Themelis, Andreas and Stella, Lorenzo and Patrinos, Panagiotis}, + journal={Mathematical Programming}, + volume={213}, + number={1}, + pages={433--471}, + year={2025}, + url={https://doi.org/10.1007/s10107-024-02143-7} } \ No newline at end of file diff --git a/docs/src/guide/implemented_algorithms.md b/docs/src/guide/implemented_algorithms.md index a14fe6b9..5059ee8d 100644 --- a/docs/src/guide/implemented_algorithms.md +++ b/docs/src/guide/implemented_algorithms.md @@ -27,6 +27,7 @@ Algorithm | Assumptions | Oracle | Implementation | References Proximal gradient | ``f`` locally smooth | ``\nabla f``, ``\operatorname{prox}_{\gamma g}`` | [`ForwardBackward`](@ref) | [Lions1979](@cite), [DeMarchi2024](@cite) Douglas-Rachford | | ``\operatorname{prox}_{\gamma f}``, ``\operatorname{prox}_{\gamma g}`` | [`DouglasRachford`](@ref) | [Eckstein1992](@cite) Fast proximal gradient | ``f`` convex, smooth, ``g`` convex | ``\nabla f``, ``\operatorname{prox}_{\gamma g}`` | [`FastForwardBackward`](@ref) | [Tseng2008](@cite), [Beck2009](@cite) +AdaPGM | ``f`` convex, smooth, ``g`` convex | ``\nabla f``, ``\operatorname{prox}_{\gamma g}`` | [`AdaptiveProximalGradient`](@ref) | [Latafat2025](@cite) PANOC | ``f`` smooth | ``\nabla f``, ``\operatorname{prox}_{\gamma g}`` | [`PANOC`](@ref) | [Stella2017](@cite) ZeroFPR | ``f`` smooth | ``\nabla f``, ``\operatorname{prox}_{\gamma g}`` | [`ZeroFPR`](@ref) | [Themelis2018](@cite) Douglas-Rachford line-search | ``f`` smooth | ``\operatorname{prox}_{\gamma f}``, ``\operatorname{prox}_{\gamma g}`` | [`DRLS`](@ref) | [Themelis2022](@cite) @@ -39,6 +40,8 @@ ProximalAlgorithms.DouglasRachford ProximalAlgorithms.DouglasRachfordIteration ProximalAlgorithms.FastForwardBackward ProximalAlgorithms.FastForwardBackwardIteration +ProximalAlgorithms.AdaptiveProximalGradient +ProximalAlgorithms.AdaptiveProximalGradientIteration ProximalAlgorithms.PANOC ProximalAlgorithms.PANOCIteration ProximalAlgorithms.ZeroFPR @@ -74,6 +77,7 @@ Algorithm | Assumptions | Oracle | Implementation | References Chambolle-Pock | ``f\equiv 0``, ``g, h`` convex, ``L`` linear operator | ``\operatorname{prox}_{\gamma g}``, ``\operatorname{prox}_{\gamma h}``, ``L``, ``L^*`` | [`ChambollePock`](@ref) | [Chambolle2011](@cite) Vu-Condat | ``f`` convex and smooth, ``g, h`` convex, ``L`` linear operator | ``\nabla f``, ``\operatorname{prox}_{\gamma g}``, ``\operatorname{prox}_{\gamma h}``, ``L``, ``L^*`` | [`VuCondat`](@ref) | [Vu2013](@cite), [Condat2013](@cite) AFBA | ``f`` convex and smooth, ``g, h`` convex, ``L`` linear operator | ``\nabla f``, ``\operatorname{prox}_{\gamma g}``, ``\operatorname{prox}_{\gamma h}``, ``L``, ``L^*`` | [`AFBA`](@ref) | [Latafat2017](@cite) +AdaPDM | ``f`` convex and smooth, ``g, h`` convex, ``L`` linear operator | ``\nabla f``, ``\operatorname{prox}_{\gamma g}``, ``\operatorname{prox}_{\gamma h}``, ``L``, ``L^*`` | [`AdaptivePrimalDual`](@ref) | [Latafat2025](@cite) ```@docs ProximalAlgorithms.ChambollePock @@ -82,4 +86,6 @@ ProximalAlgorithms.VuCondat ProximalAlgorithms.VuCondatIteration ProximalAlgorithms.AFBA ProximalAlgorithms.AFBAIteration +ProximalAlgorithms.AdaptivePrimalDual +ProximalAlgorithms.AdaptivePrimalDualIteration ``` diff --git a/src/ProximalAlgorithms.jl b/src/ProximalAlgorithms.jl index e248613b..a9ba8bbe 100644 --- a/src/ProximalAlgorithms.jl +++ b/src/ProximalAlgorithms.jl @@ -43,6 +43,7 @@ end include("utilities/fb_tools.jl") include("utilities/iteration_tools.jl") +nan_to_zero(v) = ifelse(isnan(v), zero(v), v) # acceleration utilities @@ -135,5 +136,7 @@ include("algorithms/davis_yin.jl") include("algorithms/li_lin.jl") include("algorithms/sfista.jl") include("algorithms/panocplus.jl") +include("algorithms/adapgm.jl") +include("algorithms/adapdm.jl") end # module diff --git a/src/algorithms/adapdm.jl b/src/algorithms/adapdm.jl new file mode 100644 index 00000000..a3115c86 --- /dev/null +++ b/src/algorithms/adapdm.jl @@ -0,0 +1,264 @@ +# Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms +# for convex optimization under local Lipschitz continuity of the gradient", +# Mathematical Programming, vol. 213, pp. 433-471 (2025). + +using Base.Iterators +using ProximalAlgorithms.IterationTools +using ProximalCore: Zero, convex_conjugate +using LinearAlgebra +using Printf + +# TODO implement AdaPDMplus (Algorithm 3.2 of [1]) + +# TODO Default Lopnorm = opnorm(L) +# For a dense L this is a full SVD, which can be expensive at scale. It also fails or is +# impractical for sparse or matrix-free operators. Document that users can pass Lopnorm +# themselves, or use a power-iteration estimate as the default (with a safety factor, since +# an underestimate breaks the stepsize bound). +""" + AdaptivePrimalDualIteration(; ) + +Iterator implementing the adaptive three-term primal-dual algorithm (AdaPDM, see [1]). + +This iterator solves convex optimization problems of the form + + minimize f(x) + g(x) + h(L x), + +where `f` is smooth, `g` and `h` are possibly nonsmooth, and `L` is a linear mapping. + +Points `x0` and `y0` are the initial primal and dual iterates, respectively. +If unspecified, functions `f`, `g`, and `h` default to the identically zero +function, and `L` defaults to the identity. + +The iterator implements Algorithm 3.1 of [1]. A linesearch variant that does not require +the linear operator norm is given by Algorithm 3.2 of [1], not yet implemented. + +See also: [`AdaptivePrimalDual`](@ref). + +# Arguments +- `x0`: initial primal point. +- `y0`: initial dual point. +- `f=Zero()`: smooth objective term. +- `g=Zero()`: proximable objective term. +- `h=Zero()`: proximable objective term. +- `L=I`: linear operator (e.g. a matrix). +- `Lopnorm=opnorm(L)`: operator norm of `L`. +- `t=1`: positive algorithm parameter. +- `epsil=1e-6`: positive algorithm parameter. +- `nu=max(1.2,1+2*epsil)`: algorithm parameter such that `nu > 1 + epsil`. + +# References +1. Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient", Mathematical Programming, vol. 213, pp. 433-471 (2025). +""" +Base.@kwdef struct AdaptivePrimalDualIteration{R<:Real,Tx,Ty,Tf,Tg,Th,TL,Tnormop} + f::Tf = Zero() + g::Tg = Zero() + h::Th = Zero() + L::TL = I + x0::Tx + y0::Ty + Lopnorm::Tnormop = LinearAlgebra.opnorm(L) # TODO not provided? then linesearch variant AdaPDMplus + t::R = real(eltype(x0))(1) + epsil::R = real(eltype(x0))(1e-6) + nu::R = max(real(eltype(x0))(1.2), 1+2*epsil) +end + +Base.IteratorSize(::Type{<:AdaptivePrimalDualIteration}) = Base.IsInfinite() + +Base.@kwdef mutable struct AdaptivePrimalDualState{R,Tx,Ty} + x::Tx # primal iterate + y::Ty # dual iterate + xbar::Tx + ybar::Ty + grad_f_x::Tx # gradient of f at x + grad_f_xold::Tx # gradient of f at previous x + L_x::Ty + L_xold::Ty + gamma::R + gamma_old::R + tmp_x::Tx + tmp_y::Ty + FPR_x::Tx # fixed-point x-residual + FPR_y::Ty # fixed-point y-residual +end + +function Base.iterate(iter::AdaptivePrimalDualIteration) + iter.Lopnorm > 0 || throw(ArgumentError("`Lopnorm` must be positive, got $iter.Lopnorm")) + iter.t > 0 || throw(ArgumentError("`t` must be positive, got $iter.t")) + iter.epsil > 0 || throw(ArgumentError("`epsil` must be positive, got $iter.epsil")) + iter.nu > 1 + iter.epsil || throw(ArgumentError("`nu` must satisfy nu > 1 + epsil, got nu = $iter.nu, epsil = $iter.epsil")) + + x = copy(iter.x0) + y = copy(iter.y0) + L_x = similar(y) + mul!(L_x, iter.L, x) + R = real(eltype(x)) + gamma = R(1/(2 * iter.nu * iter.t * iter.Lopnorm)) + + state = AdaptivePrimalDualState( + x=x, + y=y, + xbar=copy(x), + ybar=copy(y), + grad_f_x=similar(x), + grad_f_xold=similar(x), + L_x=L_x, + L_xold=similar(y), + gamma=gamma, + gamma_old=copy(gamma), + tmp_x=similar(x), + tmp_y=similar(y), + FPR_x=similar(x), + FPR_y=similar(y), + ) + + _, state.grad_f_x = value_and_gradient(iter.f, state.x) + + # x-update + mul!(state.tmp_x, iter.L', state.ybar) + state.tmp_x .+= state.grad_f_x + state.tmp_x .*= (- state.gamma) + state.tmp_x .+= state.x + prox!(state.xbar, iter.g, state.tmp_x, state.gamma) + + # residuals + state.FPR_x .= state.xbar .- state.x + state.FPR_y .= Inf + + return state, state +end + +# stepsize rule as defined in Algorithm 3.1 of [1] +function get_stepsizes(iter::AdaptivePrimalDualIteration, gamma, x, grad_x, gamma_old, x_old, grad_x_old) + return get_stepsizes(iter, gamma, gamma_old, x-x_old, grad_x-grad_x_old) +end + +function get_stepsizes(iter::AdaptivePrimalDualIteration, gamma, gamma_old, delta_x, delta_grad_x) + R = real(eltype(delta_x)) + t = iter.t + nu = iter.nu + epsil = iter.epsil + Lopnorm = iter.Lopnorm + + # ℓ_k and c_k from (1.3) of [1] + dx2 = norm(delta_x)^2 + l = real(dot(delta_grad_x, delta_x)) / dx2 |> nan_to_zero + lc = norm(delta_grad_x)^2 / dx2 |> nan_to_zero + + # δ_k and ξ_k + delta = gamma * (gamma*lc - l) |> nan_to_zero + xi = (t*gamma*Lopnorm)^2 + + # γ_k+1 + cc1 = gamma * sqrt(1 + gamma/gamma_old) + cc2 = 1/(2*nu*t*Lopnorm) + cc0 = 1-4*xi*(1+epsil)^2 + cc3 = gamma*sqrt(cc0/(2*(1+epsil)*(sqrt(delta^2 + xi*cc0) + delta))) + gamma_new = R(min(cc1, cc2, cc3)) + return gamma_new, gamma +end + +function Base.iterate( + iter::AdaptivePrimalDualIteration, + state::AdaptivePrimalDualState, +) + state.grad_f_xold .= state.grad_f_x + _, state.grad_f_x = value_and_gradient(iter.f, state.xbar) + state.L_xold .= state.L_x + mul!(state.L_x, iter.L, state.xbar) + + # stepsizes + state.tmp_x .= state.xbar .- state.x # temporary + state.FPR_x .= state.grad_f_x .- state.grad_f_xold # temporary + state.gamma, state.gamma_old = get_stepsizes(iter, state.gamma, state.gamma_old, state.tmp_x, state.FPR_x) + sigma = state.gamma * iter.t^2 + + # x-residual + # (tmp_x = xbar - x, FPR_x = grad_x - grad_xold at this point) + state.FPR_x .-= state.tmp_x ./ state.gamma_old + + state.x .= state.xbar + state.y .= state.ybar + + # y-update + rho = state.gamma / state.gamma_old + state.tmp_y .= (1+rho) .* state.L_x .- rho .* state.L_xold + state.tmp_y .*= sigma + state.tmp_y .+= state.y + prox!(state.ybar, convex_conjugate(iter.h), state.tmp_y, sigma) + + # y-residual + state.FPR_y .= (state.tmp_y .- state.ybar) ./ sigma .- state.L_x + + # x-update + mul!(state.tmp_x, iter.L', state.ybar) + state.tmp_x .+= state.grad_f_x + state.tmp_x .*= (- state.gamma) + state.tmp_x .+= state.x + prox!(state.xbar, iter.g, state.tmp_x, state.gamma) + + return state, state +end + +default_stopping_criterion(tol, ::AdaptivePrimalDualIteration, state::AdaptivePrimalDualState) = + sqrt(norm(state.FPR_x)^2 + norm(state.FPR_y)^2) <= tol +default_solution(::AdaptivePrimalDualIteration, state::AdaptivePrimalDualState) = (state.x, state.ybar) +default_display(it, ::AdaptivePrimalDualIteration, state::AdaptivePrimalDualState) = + @printf("%6d | %7.4e %7.4e\n", it, norm(state.FPR_x), norm(state.FPR_y)) + +""" + AdaptivePrimalDual(; ) + +Constructs the adaptive linesearch-free three-term primal-dual algorithm (AdaPDM, see [1]). + +This algorithm solves convex optimization problems of the form + + minimize f(x) + g(x) + h(L x), + +where `f` is smooth, `g` and `h` are possibly nonsmooth, and `L` is a linear mapping. + +The returned object has type `IterativeAlgorithm{AdaptivePrimalDualIteration}`, +and can be called with the problem's arguments to trigger its solution. +The solution is returned as an (x, y) tuple. + +See also: [`AdaptivePrimalDualIteration`](@ref), [`IterativeAlgorithm`](@ref). + +# Arguments +- `maxit::Int=10_000`: maximum number of iteration +- `tol=1e-5`: tolerance for the default stopping criterion +- `stop::Function`: termination condition, `stop(::T, state)` should return `true` when to stop the iteration +- `solution::Function`: solution mapping, `solution(::T, state)` should return the identified solution +- `verbose::Bool=false`: whether the algorithm state should be displayed +- `freq::Int=100`: every how many iterations to display the algorithm state +- `display::Function`: display function, `display(::Int, ::T, state)` should display a summary of the iteration state +- `kwargs...`: additional keyword arguments to pass on to the `AdaptivePrimalDualIteration` constructor upon call + +# References +1. Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient", Mathematical Programming, vol. 213, pp. 433-471 (2025). +""" +AdaptivePrimalDual(; + maxit=10_000, + tol=1e-5, + stop=(iter, state) -> default_stopping_criterion(tol, iter, state), + solution=default_solution, + verbose=false, + freq=100, + display=default_display, + kwargs..., +) = IterativeAlgorithm( + AdaptivePrimalDualIteration; + maxit, + stop, + solution, + verbose, + freq, + display, + kwargs..., +) + + +# Aliases + +const AdaPDMState = AdaptivePrimalDualState +const AdaPDMIteration = AdaptivePrimalDualIteration +const AdaPDM = AdaptivePrimalDual \ No newline at end of file diff --git a/src/algorithms/adapgm.jl b/src/algorithms/adapgm.jl new file mode 100644 index 00000000..9c6ffdab --- /dev/null +++ b/src/algorithms/adapgm.jl @@ -0,0 +1,174 @@ +# Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization +# under local Lipschitz continuity of the gradient," Mathematical Programming, vol. 213, pp. 433–471 (2025). + +using Base.Iterators +using ProximalAlgorithms.IterationTools +using ProximalCore: Zero +using LinearAlgebra +using Printf + +""" + AdaptiveProximalGradientIteration(; ) + +Iterator implementing the adaptive proximal-gradient method [1]. + +This iterator solves convex optimization problems of the form + + minimize f(x) + g(x), + +where `f` is smooth and `g` is prox-friendly. + +See also: [`AdaptiveProximalGradient`](@ref). + +# Arguments +- `x0`: initial point. +- `f=Zero()`: smooth objective term. +- `g=Zero()`: proximable objective term. +- `Lf=nothing`: Lipschitz constant estimate of the gradient of `f`. +- `gamma=nothing`: stepsize, defaults to `1/Lf` if `Lf` is set, and `nothing` otherwise. + +# References +1. Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient" (2025). +""" +Base.@kwdef struct AdaptiveProximalGradientIteration{Tx,Tf,Tg,TLf,Tgamma} + f::Tf = Zero() + g::Tg = Zero() + x0::Tx + Lf::TLf = nothing + gamma::Tgamma = Lf === nothing ? nothing : (1 / Lf) +end + +Base.IteratorSize(::Type{<:AdaptiveProximalGradientIteration}) = Base.IsInfinite() + +Base.@kwdef mutable struct AdaptiveProximalGradientState{R,Tx} + x::Tx # iterate + f_x::R # value f at x + grad_f_x::Tx # gradient of f at x + gamma::R # stepsize parameter of forward and backward steps + y::Tx # forward point + z::Tx # forward-backward point + f_z::R # value f at z + g_z::R # value of g at z + grad_f_z::Tx # gradient of f at z + res::Tx # fixed-point residual at iterate + gamma_old::R # previous stepsize parameter +end + +function initial_stepsize(f, g, x, grad_f_x) + R = real(eltype(x)) + L = lower_bound_smoothness_constant(f, I, x, grad_f_x) + return (isfinite(L) && L > 0) ? 1/R(L) : one(R) +end + +function Base.iterate(iter::AdaptiveProximalGradientIteration) + x = copy(iter.x0) + f_x, grad_f_x = value_and_gradient(iter.f, x) + gamma = iter.gamma === nothing ? initial_stepsize(iter.f, iter.g, x, grad_f_x) : iter.gamma + y = x - gamma .* grad_f_x + z, g_z = prox(iter.g, y, gamma) + f_z, grad_f_z = value_and_gradient(iter.f, z) + state = AdaptiveProximalGradientState( + x=x, + f_x=f_x, + grad_f_x=grad_f_x, + gamma=gamma, + y=y, + z=z, + f_z=f_z, + g_z=g_z, + grad_f_z=grad_f_z, + res=x - z, + gamma_old=gamma, + ) + return state, state +end + +# stepsize rule as defined in Algorithm 2.1 of [1] +function get_stepsize(gamma, x, grad_x, gamma_old, x_old, grad_x_old) + LC = norm(grad_x - grad_x_old)^2 / norm(x - x_old)^2 |> nan_to_zero + L = real(dot(grad_x - grad_x_old, x - x_old)) / norm(x - x_old)^2 |> nan_to_zero + D = gamma * max(0, gamma*LC - L) + gamma_new = gamma * min(sqrt(1 + gamma / gamma_old), 1 / (2 * sqrt(D))) + return gamma_new, gamma +end + +function Base.iterate( + iter::AdaptiveProximalGradientIteration, + state::AdaptiveProximalGradientState{R,Tx}, +) where {R,Tx} + # estimate stepsize + state.gamma, state.gamma_old = get_stepsize(state.gamma, state.z, state.grad_f_z, state.gamma_old, state.x, state.grad_f_x) + # shift + state.x .= state.z + state.grad_f_x .= state.grad_f_z + # compute new point + state.y .= state.x .- state.gamma .* state.grad_f_x + state.g_z = prox!(state.z, iter.g, state.y, state.gamma) + state.f_z, state.grad_f_z = value_and_gradient(iter.f, state.z) + state.res .= state.x .- state.z + return state, state +end + +default_stopping_criterion( + tol, + ::AdaptiveProximalGradientIteration, + state::AdaptiveProximalGradientState, +) = norm(state.res, Inf) / state.gamma <= tol +default_solution(::AdaptiveProximalGradientIteration, state::AdaptiveProximalGradientState) = state.z +default_display(it, ::AdaptiveProximalGradientIteration, state::AdaptiveProximalGradientState) = + @printf("%5d | %.3e | %.3e\n", it, state.gamma, norm(state.res, Inf) / state.gamma) + +""" + AdaptiveProximalGradient(; ) + +Constructs the adaptive proximal-gradient algorithm [1]. + +This algorithm solves convex optimization problems of the form + + minimize f(x) + g(x), + +where `f` is locally smooth. + +The returned object has type `IterativeAlgorithm{AdaptiveProximalGradientIteration}`, +and can be called with the problem's arguments to trigger its solution. + +See also: [`AdaptiveProximalGradientIteration`](@ref), [`IterativeAlgorithm`](@ref). + +# Arguments +- `maxit::Int=10_000`: maximum number of iteration +- `tol::1e-8`: tolerance for the default stopping criterion +- `stop::Function`: termination condition, `stop(::T, state)` should return `true` when to stop the iteration +- `solution::Function`: solution mapping, `solution(::T, state)` should return the identified solution +- `verbose::Bool=false`: whether the algorithm state should be displayed +- `freq::Int=100`: every how many iterations to display the algorithm state +- `display::Function`: display function, `display(::Int, ::T, state)` should display a summary of the iteration state +- `kwargs...`: additional keyword arguments to pass on to the `AdaptiveProximalGradientIteration` constructor upon call + +# References +1. Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient" (2025). +""" +AdaptiveProximalGradient(; + maxit=10_000, + tol=1e-8, + stop=(iter, state) -> default_stopping_criterion(tol, iter, state), + solution=default_solution, + verbose=false, + freq=100, + display=default_display, + kwargs..., +) = IterativeAlgorithm( + AdaptiveProximalGradientIteration; + maxit, + stop, + solution, + verbose, + freq, + display, + kwargs..., +) + +# Aliases + +const AdaPGMState = AdaptiveProximalGradientState +const AdaPGMIteration = AdaptiveProximalGradientIteration +const AdaPGM = AdaptiveProximalGradient \ No newline at end of file diff --git a/test/problems/test_elasticnet.jl b/test/problems/test_elasticnet.jl index 274d8ea4..3c3e34d4 100644 --- a/test/problems/test_elasticnet.jl +++ b/test/problems/test_elasticnet.jl @@ -24,7 +24,7 @@ using DifferentiationInterface: AutoZygote cost = ProximalAlgorithms.AutoDifferentiable(x -> (norm(A * x - b)^2) / 2, AutoZygote()) - L = opnorm(A)^2 + Lf = opnorm(A)^2 x_star = T[-0.6004983388704322, 0.0, 0.0, 0.195182724252491, 0.764119601328903] @@ -35,7 +35,7 @@ using DifferentiationInterface: AutoZygote x0 = zeros(T, n) x0_backup = copy(x0) solver = ProximalAlgorithms.DavisYin(tol = R(1e-6)) - xf_dys, it_dys = @inferred solver(x0 = x0, f = cost, g = reg1, h = reg2, Lf = L) + xf_dys, it_dys = @inferred solver(x0 = x0, f = cost, g = reg1, h = reg2, Lf = Lf) @test eltype(xf_dys) == T @test norm(xf_dys - x_star, Inf) <= 1e-3 @test it_dys <= 140 @@ -46,7 +46,7 @@ using DifferentiationInterface: AutoZygote x0 = randn(T, n) x0_backup = copy(x0) solver = ProximalAlgorithms.DavisYin(tol = R(1e-6)) - xf_dys, it_dys = @inferred solver(x0 = x0, f = cost, g = reg1, h = reg2, Lf = L) + xf_dys, it_dys = @inferred solver(x0 = x0, f = cost, g = reg1, h = reg2, Lf = Lf) @test eltype(xf_dys) == T @test norm(xf_dys - x_star, Inf) <= 1e-3 @test x0 == x0_backup @@ -105,4 +105,86 @@ using DifferentiationInterface: AutoZygote @test y0 == y0_backup end + + adapdm_test_params = [(1.0, 1e-6, 450), (2.0, 1e-6, 450), (1.0, 1e-3, 450), (1.0, 1.0, 1000)] + + @testset "AdaPDM" for (t, epsil, maxit) in adapdm_test_params + tmptol = T in [Float32, ComplexF32] ? R(1e-5) : R(1e-7) + + # with known initial iterates + x0 = zeros(T, n) + x0_backup = copy(x0) + y0 = zeros(T, m) + y0_backup = copy(y0) + + solver = ProximalAlgorithms.AdaptivePrimalDual(tol=tmptol) + (x, y), it = solver( + x0=x0, + y0=y0, + f=ProximalAlgorithms.AutoDifferentiable(reg2, AutoZygote()), + g=reg1, + h=loss, + L=A, + t=t, + epsil=epsil, + ) + @test eltype(x) == T + @test eltype(y) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it <= maxit + @test x0 == x0_backup + @test y0 == y0_backup + + # with random initial iterates + x0 = randn(T, n) + x0_backup = copy(x0) + y0 = randn(T, m) + y0_backup = copy(y0) + + solver = ProximalAlgorithms.AdaptivePrimalDual(tol=tmptol) + (x, y), it = solver( + x0=x0, + y0=y0, + f=ProximalAlgorithms.AutoDifferentiable(reg2, AutoZygote()), + g=reg1, + h=loss, + L=A, + t=t, + epsil=epsil, + ) + @test eltype(x) == T + @test eltype(y) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it <= maxit + @test x0 == x0_backup + @test y0 == y0_backup + + # with explicit Lopnorm::Float64 to test type stability + (x, y), it = solver( + x0=x0, + y0=y0, + f=ProximalAlgorithms.AutoDifferentiable(reg2, AutoZygote()), + g=reg1, + h=loss, + L=A, + t=t, + epsil=epsil, + Lopnorm=Float64(2*opnorm(A)), + ) + @test eltype(x) == T + @test eltype(y) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it <= 2*maxit + @test x0 == x0_backup + @test y0 == y0_backup + + # input arguments + @test_throws MethodError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, epsil=0) + @test_throws MethodError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, t=0) + @test_throws MethodError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, nu=1) + @test_throws ArgumentError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, epsil=R(0)) + @test_throws ArgumentError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, t=R(0)) + @test_throws ArgumentError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, nu=R(1)) + @test_throws ArgumentError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, Lopnorm=0) + end end diff --git a/test/problems/test_lasso_small.jl b/test/problems/test_lasso_small.jl index 155846a6..2cdaa2fa 100644 --- a/test/problems/test_lasso_small.jl +++ b/test/problems/test_lasso_small.jl @@ -134,6 +134,28 @@ using ProximalAlgorithms: @test x0 == x0_backup end + @testset "AdaPGM (with Lf)" begin + x0 = zeros(T, n) + x0_backup = copy(x0) + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) + x, it = @inferred solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= TOL + @test it < 30 + @test x0 == x0_backup + end + + @testset "AdaPGM (w/o Lf)" begin + x0 = zeros(T, n) + x0_backup = copy(x0) + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) + x, it = @inferred solver(x0 = x0, f = fA_autodiff, g = g) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= TOL + @test it < 40 + @test x0 == x0_backup + end + @testset "ZeroFPR (fixed step)" begin x0 = zeros(T, n) x0_backup = copy(x0) @@ -343,6 +365,44 @@ using ProximalAlgorithms: @test x0 == x0_backup end + @testset "AdaPDM" begin + x0 = zeros(T, n) + x0_backup = copy(x0) + solver = ProximalAlgorithms.AdaptivePrimalDual(tol = TOL) + (x_adapdm, y_adapdm), it_adapdm = @inferred solver( + x0 = x0, + y0 = zeros(T, n), + f = fA_autodiff, + g = g, + ) + @test eltype(x_adapdm) == T + @test eltype(y_adapdm) == T + @test norm(x_adapdm - x_star, Inf) <= 1e-4 + @test it_adapdm <= 130 + @test x0 == x0_backup + + solver = ProximalAlgorithms.AdaPDM(tol = TOL) + (x_adapdm, y_adapdm), it_adapdm = @inferred solver( + x0 = x0, + y0 = zeros(T, n), + f = fA_autodiff, + h = g, + ) + @test eltype(x_adapdm) == T + @test eltype(y_adapdm) == T + @test norm(x_adapdm - x_star, Inf) <= 1e-4 + @test it_adapdm <= 3900 + @test x0 == x0_backup + + (x_adapdm, y_adapdm), it_adapdm = + @inferred solver(x0 = x0, y0 = zeros(T, m), h = f_prox, L = A, g = g) + @test eltype(x_adapdm) == T + @test eltype(y_adapdm) == T + @test norm(x_adapdm - x_star, Inf) <= 1e-4 + @test it_adapdm <= 700 + @test x0 == x0_backup + end + @testset "SFISTA" begin x0 = zeros(T, n) x0_backup = copy(x0) diff --git a/test/problems/test_lasso_small_strongly_convex.jl b/test/problems/test_lasso_small_strongly_convex.jl index d2d16755..32229624 100644 --- a/test/problems/test_lasso_small_strongly_convex.jl +++ b/test/problems/test_lasso_small_strongly_convex.jl @@ -143,6 +143,24 @@ using ProximalAlgorithms @test x0 == x0_backup end + @testset "AdaPGM (with Lf)" begin + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) + y, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf) + @test eltype(y) == T + @test norm(y - x_star, Inf) <= TOL + @test it < 35 + @test x0 == x0_backup + end + + @testset "AdaPGM (w/o Lf)" begin + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) + y, it = solver(x0 = x0, f = fA_autodiff, g = g) + @test eltype(y) == T + @test norm(y - x_star, Inf) <= TOL + @test it < 40 + @test x0 == x0_backup + end + @testset "DRLS" begin solver = ProximalAlgorithms.DRLS(tol=TOL) v, it = solver(x0=x0, f=fA_prox, g=g, mf=mf) @@ -161,6 +179,38 @@ using ProximalAlgorithms @test x0 == x0_backup end + @testset "AdaPDM" begin + solver = ProximalAlgorithms.AdaptivePrimalDual(tol = TOL) + (v, vy), it = solver( + x0 = x0, + y0 = zeros(T, length(x0)), + f = fA_autodiff, + g = g, + ) + @test eltype(v) == T + @test norm(v - x_star, Inf) <= 5*TOL + @test it <= 50 + @test x0 == x0_backup + + (v, vy), it = solver( + x0 = x0, + y0 = zeros(T, length(x0)), + f = fA_autodiff, + h = g, + ) + @test eltype(v) == T + @test norm(v - x_star, Inf) <= 5*TOL + @test it <= 100 + @test x0 == x0_backup + + (v, vy), it = solver(x0 = x0, y0 = zeros(T, length(x0)), h = fA_prox, g = g) + @test eltype(v) == T + @test eltype(vy) == T + @test norm(v - x_star, Inf) <= 5*TOL + @test it <= 140 + @test x0 == x0_backup + end + @testset "ZeroFPR (fixed, nonmonotone)" begin solver = ProximalAlgorithms.ZeroFPR(tol = TOL, monotonicity = T(0.2)) y, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf=Lf) diff --git a/test/problems/test_linear_programs.jl b/test/problems/test_linear_programs.jl index 54a908a3..400f11ba 100644 --- a/test/problems/test_linear_programs.jl +++ b/test/problems/test_linear_programs.jl @@ -190,10 +190,26 @@ end @test it <= maxit - @assert norm(xf - x_star) <= 1e2 * tol + @test norm(xf - x_star) <= 1e2 * tol @test x0 == x0_backup + end + + @testset "AdaPDM" begin + + f = ProximalAlgorithms.AutoDifferentiable(x -> dot(c, x), AutoZygote()) + g = IndNonnegative() + x0 = zeros(T, n) + x0_backup = copy(x0) + + solver = ProximalAlgorithms.AdaPDM(tol = tol, maxit = maxit) + (v, vy), it = solver(x0 = x0, y0=zeros(T, n), f = f, g = g, h = IndAffine(A, b)) + + @test eltype(v) == T + @test it <= 25_000 + @test norm(v - x_star) <= 1e2 * tol + @test x0 == x0_backup end end diff --git a/test/problems/test_sparse_logistic_small.jl b/test/problems/test_sparse_logistic_small.jl index 19e5de69..78f8f8a3 100644 --- a/test/problems/test_sparse_logistic_small.jl +++ b/test/problems/test_sparse_logistic_small.jl @@ -87,6 +87,36 @@ using LinearAlgebra @test x0 == x0_backup end + @testset "AdaptiveProximalGradient (w/o Lf)" begin + x0 = zeros(T, n) + x0_backup = copy(x0) + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) + x, it = solver(x0 = x0, f = fA_autodiff, g = g) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it < 30 + @test x0 == x0_backup + end + + @testset "AdaPDM" begin + x0 = zeros(T, n) + x0_backup = copy(x0) + solver = ProximalAlgorithms.AdaPDM(tol = TOL) + (x, vy), it = solver(x0 = x0, y0=zeros(T,n), f = fA_autodiff, g = g) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it < 280 + @test x0 == x0_backup + + x0 = zeros(T, n) + x0_backup = copy(x0) + (x, vy), it = solver(x0 = x0, y0=zeros(T,n), f = fA_autodiff, h = g) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it < 280 + @test x0 == x0_backup + end + @testset "ZeroFPR (adaptive step)" begin x0 = zeros(T, n) x0_backup = copy(x0) diff --git a/test/problems/test_verbose.jl b/test/problems/test_verbose.jl index ce2da757..25688f02 100644 --- a/test/problems/test_verbose.jl +++ b/test/problems/test_verbose.jl @@ -77,6 +77,24 @@ using LinearAlgebra @test it < 200 end + @testset "AdaptiveProximalGradient" begin + # with Lf + x0 = zeros(T, n) + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL, verbose=true) + x, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= TOL + @test it < 35 + + # w/o Lf + x0 = zeros(T, n) + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL, verbose=true) + x, it = solver(x0 = x0, f = fA_autodiff, g = g) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= TOL + @test it < 40 + end + @testset "ZeroFPR" begin # ZeroFPR/Nonadaptive @@ -160,4 +178,20 @@ using LinearAlgebra end + @testset "AdaPDM" begin + x0 = zeros(T, n) + solver = ProximalAlgorithms.AdaPDM(tol=TOL, verbose=true) + (x, y), it = solver(x0=x0, y0=zeros(T, n), g=fA_prox, h=g) + + @test eltype(x) == T + @test norm(x - x_star, Inf) <= 5*TOL + @test it < 250 + + solver = ProximalAlgorithms.AdaPDM(tol=TOL, verbose=true, freq=42) + (x, y), it = solver(x0=x0, y0=zeros(T, n), g=fA_prox, h=g) + + @test eltype(x) == T + @test norm(x - x_star, Inf) <= 5*TOL + @test it < 250 + end end