From 8b2304096456215ce08652568a441e0c45c7a6b8 Mon Sep 17 00:00:00 2001 From: Alberto De Marchi Date: Tue, 27 Oct 2026 11:46:42 +0100 Subject: [PATCH 1/8] added AdaPGM with tests --- src/ProximalAlgorithms.jl | 1 + src/algorithms/adapgm.jl | 166 ++++++++++++++++++ test/problems/test_lasso_small.jl | 22 +++ .../test_lasso_small_strongly_convex.jl | 18 ++ test/problems/test_sparse_logistic_small.jl | 11 ++ test/problems/test_verbose.jl | 18 ++ 6 files changed, 236 insertions(+) create mode 100644 src/algorithms/adapgm.jl diff --git a/src/ProximalAlgorithms.jl b/src/ProximalAlgorithms.jl index e248613b..a14f5dec 100644 --- a/src/ProximalAlgorithms.jl +++ b/src/ProximalAlgorithms.jl @@ -135,5 +135,6 @@ include("algorithms/davis_yin.jl") include("algorithms/li_lin.jl") include("algorithms/sfista.jl") include("algorithms/panocplus.jl") +include("algorithms/adapgm.jl") end # module diff --git a/src/algorithms/adapgm.jl b/src/algorithms/adapgm.jl new file mode 100644 index 00000000..6f75b979 --- /dev/null +++ b/src/algorithms/adapgm.jl @@ -0,0 +1,166 @@ +# Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization +# under local Lipschitz continuity of the gradient," Mathematical Programming, vol. 213, pp. 433–471 (2025). + +using Base.Iterators +using ProximalAlgorithms.IterationTools +using ProximalCore: Zero +using LinearAlgebra +using Printf + +""" + AdaptiveProximalGradientIteration(; ) + +Iterator implementing the adaptive proximal-gradient method [1]. + +This iterator solves convex optimization problems of the form + + minimize f(x) + g(x), + +where `f` is smooth. + +See also: [`AdaptiveProximalGradient`](@ref). + +# Arguments +- `x0`: initial point. +- `f=Zero()`: smooth objective term. +- `g=Zero()`: proximable objective term. +- `Lf=nothing`: Lipschitz constant of the gradient of `f`. +- `gamma=nothing`: stepsize, defaults to `1/Lf` if `Lf` is set, and `nothing` otherwise. + +# References +1. Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient" (2025). +""" +Base.@kwdef struct AdaptiveProximalGradientIteration{Tx,Tf,Tg,TLf,Tgamma} + f::Tf = Zero() + g::Tg = Zero() + x0::Tx + Lf::TLf = nothing + gamma::Tgamma = Lf === nothing ? nothing : (1 / Lf) +end + +Base.IteratorSize(::Type{<:AdaptiveProximalGradientIteration}) = Base.IsInfinite() + +Base.@kwdef mutable struct AdaptiveProximalGradientState{R,Tx} + x::Tx # iterate + f_x::R # value f at x + grad_f_x::Tx # gradient of f at x + gamma::R # stepsize parameter of forward and backward steps + y::Tx # forward point + z::Tx # forward-backward point + f_z::R # value f at z + g_z::R # value of g at z + grad_f_z::Tx # gradient of f at z + res::Tx # fixed-point residual at iterate + gamma_old::R # previous stepsize parameter +end + +function Base.iterate(iter::AdaptiveProximalGradientIteration) + x = copy(iter.x0) + f_x, grad_f_x = value_and_gradient(iter.f, x) + gamma = + iter.gamma === nothing ? + 1 / lower_bound_smoothness_constant(iter.f, I, x, grad_f_x) : iter.gamma + y = x - gamma .* grad_f_x + z, g_z = prox(iter.g, y, gamma) + f_z, grad_f_z = value_and_gradient(iter.f, z) + state = AdaptiveProximalGradientState( + x=x, + f_x=f_x, + grad_f_x=grad_f_x, + gamma=gamma, + y=y, + z=z, + f_z=f_z, + g_z=g_z, + grad_f_z=grad_f_z, + res=x - z, + gamma_old=gamma, + ) + return state, state +end + +nan_to_zero(v) = ifelse(isnan(v), zero(v), v) + +# stepsize rule as defined in Algorithm 2.1 of [1] +function get_stepsize(gamma, x, grad_x, gamma_old, x_old, grad_x_old) + C = norm(grad_x - grad_x_old)^2 / real(dot(grad_x - grad_x_old, x - x_old)) |> nan_to_zero + L = real(dot(grad_x - grad_x_old, x - x_old)) / norm(x - x_old)^2 |> nan_to_zero + D = max(0, gamma * L * (gamma * C - 1)) + gamma_new = gamma * min(sqrt(1 + gamma / gamma_old), 1 / (2 * sqrt(D))) + return gamma_new, gamma +end + +function Base.iterate( + iter::AdaptiveProximalGradientIteration, + state::AdaptiveProximalGradientState{R,Tx}, +) where {R,Tx} + # estimate stepsize + state.gamma, state.gamma_old = get_stepsize(state.gamma, state.z, state.grad_f_z, state.gamma_old, state.x, state.grad_f_x) + # shift + state.x .= state.z + state.grad_f_x .= state.grad_f_z + # compute new point + state.y .= state.x .- state.gamma .* state.grad_f_x + state.g_z = prox!(state.z, iter.g, state.y, state.gamma) + state.f_z, state.grad_f_z = value_and_gradient(iter.f, state.z) + state.res .= state.x .- state.z + return state, state +end + +default_stopping_criterion( + tol, + ::AdaptiveProximalGradientIteration, + state::AdaptiveProximalGradientState, +) = norm(state.res, Inf) / state.gamma <= tol +default_solution(::AdaptiveProximalGradientIteration, state::AdaptiveProximalGradientState) = state.z +default_display(it, ::AdaptiveProximalGradientIteration, state::AdaptiveProximalGradientState) = + @printf("%5d | %.3e | %.3e\n", it, state.gamma, norm(state.res, Inf) / state.gamma) + +""" + AdaptiveProximalGradient(; ) + +Constructs the adaptive proximal-gradient algorithm [1]. + +This algorithm solves convex optimization problems of the form + + minimize f(x) + g(x), + +where `f` is locally smooth. + +The returned object has type `IterativeAlgorithm{AdaptiveProximalGradientIteration}`, +and can be called with the problem's arguments to trigger its solution. + +See also: [`AdaptiveProximalGradientIteration`](@ref), [`IterativeAlgorithm`](@ref). + +# Arguments +- `maxit::Int=10_000`: maximum number of iteration +- `tol::1e-8`: tolerance for the default stopping criterion +- `stop::Function`: termination condition, `stop(::T, state)` should return `true` when to stop the iteration +- `solution::Function`: solution mapping, `solution(::T, state)` should return the identified solution +- `verbose::Bool=false`: whether the algorithm state should be displayed +- `freq::Int=100`: every how many iterations to display the algorithm state +- `display::Function`: display function, `display(::Int, ::T, state)` should display a summary of the iteration state +- `kwargs...`: additional keyword arguments to pass on to the `AdaptiveProximalGradientIteration` constructor upon call + +# References +1. Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient" (2025). +""" +AdaptiveProximalGradient(; + maxit=10_000, + tol=1e-8, + stop=(iter, state) -> default_stopping_criterion(tol, iter, state), + solution=default_solution, + verbose=false, + freq=100, + display=default_display, + kwargs..., +) = IterativeAlgorithm( + AdaptiveProximalGradientIteration; + maxit, + stop, + solution, + verbose, + freq, + display, + kwargs..., +) diff --git a/test/problems/test_lasso_small.jl b/test/problems/test_lasso_small.jl index 155846a6..09fb606b 100644 --- a/test/problems/test_lasso_small.jl +++ b/test/problems/test_lasso_small.jl @@ -134,6 +134,28 @@ using ProximalAlgorithms: @test x0 == x0_backup end + @testset "AdaptiveProximalGradient (with Lf)" begin + x0 = zeros(T, n) + x0_backup = copy(x0) + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) + x, it = @inferred solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= TOL + @test it < 30 + @test x0 == x0_backup + end + + @testset "AdaptiveProximalGradient (w/o Lf)" begin + x0 = zeros(T, n) + x0_backup = copy(x0) + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) + x, it = @inferred solver(x0 = x0, f = fA_autodiff, g = g) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= TOL + @test it < 40 + @test x0 == x0_backup + end + @testset "ZeroFPR (fixed step)" begin x0 = zeros(T, n) x0_backup = copy(x0) diff --git a/test/problems/test_lasso_small_strongly_convex.jl b/test/problems/test_lasso_small_strongly_convex.jl index d2d16755..836639db 100644 --- a/test/problems/test_lasso_small_strongly_convex.jl +++ b/test/problems/test_lasso_small_strongly_convex.jl @@ -143,6 +143,24 @@ using ProximalAlgorithms @test x0 == x0_backup end + @testset "AdaptiveProximalGradient (with Lf)" begin + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) + y, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf) + @test eltype(y) == T + @test norm(y - x_star, Inf) <= TOL + @test it < 35 + @test x0 == x0_backup + end + + @testset "AdaptiveProximalGradient (w/o Lf)" begin + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) + y, it = solver(x0 = x0, f = fA_autodiff, g = g) + @test eltype(y) == T + @test norm(y - x_star, Inf) <= TOL + @test it < 40 + @test x0 == x0_backup + end + @testset "DRLS" begin solver = ProximalAlgorithms.DRLS(tol=TOL) v, it = solver(x0=x0, f=fA_prox, g=g, mf=mf) diff --git a/test/problems/test_sparse_logistic_small.jl b/test/problems/test_sparse_logistic_small.jl index 19e5de69..16762fd2 100644 --- a/test/problems/test_sparse_logistic_small.jl +++ b/test/problems/test_sparse_logistic_small.jl @@ -87,6 +87,17 @@ using LinearAlgebra @test x0 == x0_backup end + @testset "AdaptiveProximalGradient (w/o Lf)" begin + x0 = zeros(T, n) + x0_backup = copy(x0) + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) + x, it = solver(x0 = x0, f = fA_autodiff, g = g) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it < 30 + @test x0 == x0_backup + end + @testset "ZeroFPR (adaptive step)" begin x0 = zeros(T, n) x0_backup = copy(x0) diff --git a/test/problems/test_verbose.jl b/test/problems/test_verbose.jl index ce2da757..1112ef91 100644 --- a/test/problems/test_verbose.jl +++ b/test/problems/test_verbose.jl @@ -77,6 +77,24 @@ using LinearAlgebra @test it < 200 end + @testset "AdaptiveProximalGradient" begin + # with Lf + x0 = zeros(T, n) + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL, verbose=true) + x, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= TOL + @test it < 35 + + # w/o Lf + x0 = zeros(T, n) + solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL, verbose=true) + x, it = solver(x0 = x0, f = fA_autodiff, g = g) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= TOL + @test it < 40 + end + @testset "ZeroFPR" begin # ZeroFPR/Nonadaptive From bf204bd10dcabe1100fe4513da5c34c8cb53f359 Mon Sep 17 00:00:00 2001 From: Alberto De Marchi Date: Fri, 13 Nov 2026 09:31:33 +0100 Subject: [PATCH 2/8] add aliases --- src/algorithms/adapgm.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/algorithms/adapgm.jl b/src/algorithms/adapgm.jl index 6f75b979..82acde43 100644 --- a/src/algorithms/adapgm.jl +++ b/src/algorithms/adapgm.jl @@ -164,3 +164,9 @@ AdaptiveProximalGradient(; display, kwargs..., ) + +# Aliases + +const AdaPGMState = AdaptiveProximalGradientState +const AdaPGMIteration = AdaptiveProximalGradientIteration +const AdaPGM = AdaptiveProximalGradient \ No newline at end of file From cc2b9c161dd57de5436fb8c72173159052a32cc3 Mon Sep 17 00:00:00 2001 From: Alberto De Marchi Date: Sat, 14 Nov 2026 18:02:16 +0100 Subject: [PATCH 3/8] better handling initial stepsize and prevent NaN --- src/algorithms/adapgm.jl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/algorithms/adapgm.jl b/src/algorithms/adapgm.jl index 82acde43..976758a8 100644 --- a/src/algorithms/adapgm.jl +++ b/src/algorithms/adapgm.jl @@ -16,7 +16,7 @@ This iterator solves convex optimization problems of the form minimize f(x) + g(x), -where `f` is smooth. +where `f` is smooth and `g` is prox-friendly. See also: [`AdaptiveProximalGradient`](@ref). @@ -24,7 +24,7 @@ See also: [`AdaptiveProximalGradient`](@ref). - `x0`: initial point. - `f=Zero()`: smooth objective term. - `g=Zero()`: proximable objective term. -- `Lf=nothing`: Lipschitz constant of the gradient of `f`. +- `Lf=nothing`: Lipschitz constant estimate of the gradient of `f`. - `gamma=nothing`: stepsize, defaults to `1/Lf` if `Lf` is set, and `nothing` otherwise. # References @@ -54,12 +54,16 @@ Base.@kwdef mutable struct AdaptiveProximalGradientState{R,Tx} gamma_old::R # previous stepsize parameter end +function initial_stepsize(f, g, x, grad_f_x) + R = real(eltype(x)) + L = lower_bound_smoothness_constant(f, I, x, grad_f_x) + return (isfinite(L) && L > 0) ? 1/R(L) : one(R) +end + function Base.iterate(iter::AdaptiveProximalGradientIteration) x = copy(iter.x0) f_x, grad_f_x = value_and_gradient(iter.f, x) - gamma = - iter.gamma === nothing ? - 1 / lower_bound_smoothness_constant(iter.f, I, x, grad_f_x) : iter.gamma + gamma = iter.gamma === nothing ? initial_stepsize(iter.f, iter.g, x, grad_f_x) : iter.gamma y = x - gamma .* grad_f_x z, g_z = prox(iter.g, y, gamma) f_z, grad_f_z = value_and_gradient(iter.f, z) @@ -83,9 +87,9 @@ nan_to_zero(v) = ifelse(isnan(v), zero(v), v) # stepsize rule as defined in Algorithm 2.1 of [1] function get_stepsize(gamma, x, grad_x, gamma_old, x_old, grad_x_old) - C = norm(grad_x - grad_x_old)^2 / real(dot(grad_x - grad_x_old, x - x_old)) |> nan_to_zero + LC = norm(grad_x - grad_x_old)^2 / norm(x - x_old)^2 |> nan_to_zero L = real(dot(grad_x - grad_x_old, x - x_old)) / norm(x - x_old)^2 |> nan_to_zero - D = max(0, gamma * L * (gamma * C - 1)) + D = gamma * max(0, gamma*LC - L) gamma_new = gamma * min(sqrt(1 + gamma / gamma_old), 1 / (2 * sqrt(D))) return gamma_new, gamma end From 62885abeba1a1a95bc263f8e956b688ba0e356d5 Mon Sep 17 00:00:00 2001 From: Alberto De Marchi Date: Sun, 15 Nov 2026 00:31:17 +0100 Subject: [PATCH 4/8] added AdaPDM with tests --- src/ProximalAlgorithms.jl | 2 + src/algorithms/adapdm.jl | 250 ++++++++++++++++++ src/algorithms/adapgm.jl | 2 - test/problems/test_elasticnet.jl | 59 ++++- test/problems/test_lasso_small.jl | 43 ++- .../test_lasso_small_strongly_convex.jl | 36 ++- test/problems/test_linear_programs.jl | 18 +- test/problems/test_sparse_logistic_small.jl | 19 ++ test/problems/test_verbose.jl | 9 + 9 files changed, 428 insertions(+), 10 deletions(-) create mode 100644 src/algorithms/adapdm.jl diff --git a/src/ProximalAlgorithms.jl b/src/ProximalAlgorithms.jl index a14f5dec..a9ba8bbe 100644 --- a/src/ProximalAlgorithms.jl +++ b/src/ProximalAlgorithms.jl @@ -43,6 +43,7 @@ end include("utilities/fb_tools.jl") include("utilities/iteration_tools.jl") +nan_to_zero(v) = ifelse(isnan(v), zero(v), v) # acceleration utilities @@ -136,5 +137,6 @@ include("algorithms/li_lin.jl") include("algorithms/sfista.jl") include("algorithms/panocplus.jl") include("algorithms/adapgm.jl") +include("algorithms/adapdm.jl") end # module diff --git a/src/algorithms/adapdm.jl b/src/algorithms/adapdm.jl new file mode 100644 index 00000000..05a8c622 --- /dev/null +++ b/src/algorithms/adapdm.jl @@ -0,0 +1,250 @@ +# Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms +# for convex optimization under local Lipschitz continuity of the gradient", +# Mathematical Programming, vol. 213, pp. 433-471 (2025). + +using Base.Iterators +using ProximalAlgorithms.IterationTools +using ProximalCore: Zero, convex_conjugate +using LinearAlgebra +using Printf + +# TODO implement AdaPDMplus + +""" + AdaptivePrimalDualIteration(; ) + +Iterator implementing the adaptive three-term primal-dual algorithm (AdaPDM, see [1]). + +This iterator solves convex optimization problems of the form + + minimize f(x) + g(x) + h(L x), + +where `f` is smooth, `g` and `h` are possibly nonsmooth, and `L` is a linear mapping. + +Points `x0` and `y0` are the initial primal and dual iterates, respectively. +If unspecified, functions `f`, `g`, and `h` default to the identically zero +function, and `L` defaults to the identity. + +The iterator implements Algorithms 3.1 and 3.2 of [1], the latter being a linesearch +variant that does not require the linear operator norm. + +See also: [`AdaptivePrimalDual`](@ref). + +# Arguments +- `x0`: initial primal point. +- `y0`: initial dual point. +- `f=Zero()`: smooth objective term. +- `g=Zero()`: proximable objective term. +- `h=Zero()`: proximable objective term. +- `L=I`: linear operator (e.g. a matrix). +- `Lopnorm=opnorm(L)`: operator norm of `L`. +- `t=1`: positive algorithm parameter. +- `epsil=1e-6`: positive algorithm parameter. +- `nu=1.2`: algorithm parameter such that `nu > 1 + epsil`. + +# References +1. Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient", Mathematical Programming, vol. 213, pp. 433-471 (2025). +""" +Base.@kwdef struct AdaptivePrimalDualIteration{R,Tx,Ty,Tf,Tg,Th,TL,Tnormop} + f::Tf = Zero() + g::Tg = Zero() + h::Th = Zero() + L::TL = I + x0::Tx + y0::Ty + Lopnorm::Tnormop = LinearAlgebra.opnorm(L) # TODO not provided? then linesearch variant + t::R = 1.0 + epsil::R = 1e-6 + nu::R = max(1.2, 1+2*epsil) +end + +Base.IteratorSize(::Type{<:AdaptivePrimalDualIteration}) = Base.IsInfinite() + +Base.@kwdef mutable struct AdaptivePrimalDualState{R,Tx,Ty} + x::Tx # primal iterate + y::Ty # dual iterate + xbar::Tx + ybar::Ty + grad_f_x::Tx # gradient of f at x + grad_f_xold::Tx # gradient of f at previous x + L_x::Ty + L_xold::Ty + gamma::R + gamma_old::R + tmp_x::Tx + tmp_y::Ty + FPR_x::Tx # fixed-point x-residual + FPR_y::Ty # fixed-point y-residual +end + +function Base.iterate(iter::AdaptivePrimalDualIteration) + @assert isreal(iter.Lopnorm) + @assert isreal(iter.t) + @assert isreal(iter.epsil) + @assert isreal(iter.nu) + @assert iter.Lopnorm > 0 + @assert iter.t > 0 + @assert iter.epsil > 0 + @assert iter.nu > 1 + iter.epsil + + x = copy(iter.x0) + y = copy(iter.y0) + L_x = similar(y) + mul!(L_x, iter.L, x) + gamma = 1/(2 * iter.nu * iter.t * iter.Lopnorm) + + state = AdaptivePrimalDualState( + x=x, + y=y, + xbar=copy(x), + ybar=copy(y), + grad_f_x=similar(x), + grad_f_xold=similar(x), + L_x=L_x, + L_xold=similar(y), + gamma=gamma, + gamma_old=copy(gamma), + tmp_x=similar(x), + tmp_y=similar(y), + FPR_x=similar(x), + FPR_y=similar(y), + ) + + _, state.grad_f_x = value_and_gradient(iter.f, state.x) + + # x-update + mul!(state.tmp_x, iter.L', state.ybar) + state.tmp_x .+= state.grad_f_x + state.tmp_x .*= (- state.gamma) + state.tmp_x .+= state.x + prox!(state.xbar, iter.g, state.tmp_x, state.gamma) + + # residuals + state.FPR_x .= state.xbar .- state.x + state.FPR_y .= Inf + + return state, state +end + +# stepsize rule as defined in Algorithm 3.1 of [1] +function get_stepsizes(iter::AdaptivePrimalDualIteration, gamma, x, grad_x, gamma_old, x_old, grad_x_old) + t = iter.t + nu = iter.nu + epsil = iter.epsil + Lopnorm = iter.Lopnorm + + # ℓ_k and c_k from (1.3) of [1] + l = real(dot(grad_x - grad_x_old, x - x_old)) / norm(x - x_old)^2 |> nan_to_zero + lc = norm(grad_x - grad_x_old)^2 / norm(x - x_old)^2 |> nan_to_zero + + # δ_k and ξ_k + delta = gamma * (gamma*lc - l) |> nan_to_zero + xi = (t*gamma*Lopnorm)^2 + + # γ_k+1 + cc1 = gamma * sqrt(1 + gamma/gamma_old) + cc2 = 1/(2*nu*t*Lopnorm) + cc0 = 1-4*xi*(1+epsil)^2 + cc3 = gamma*sqrt(cc0/(2*(1+epsil)*(sqrt(delta^2 + xi*cc0) + delta))) + gamma_new = min(cc1, cc2, cc3) + return gamma_new, gamma +end + +function Base.iterate( + iter::AdaptivePrimalDualIteration, + state::AdaptivePrimalDualState, +) + state.grad_f_xold .= state.grad_f_x + _, state.grad_f_x = value_and_gradient(iter.f, state.xbar) + state.L_xold .= state.L_x + mul!(state.L_x, iter.L, state.xbar) + + # stepsizes + state.gamma, state.gamma_old = get_stepsizes(iter, state.gamma, state.xbar, state.grad_f_x, state.gamma_old, state.x, state.grad_f_xold) + sigma = state.gamma * iter.t^2 + + state.x .= state.xbar + state.y .= state.ybar + + # y-update + rho = state.gamma / state.gamma_old + state.tmp_y .= (1+rho) .* state.L_x - rho .* state.L_xold + state.tmp_y .*= sigma + state.tmp_y .+= state.y + prox!(state.ybar, convex_conjugate(iter.h), state.tmp_y, sigma) + + # x-update + mul!(state.tmp_x, iter.L', state.ybar) + state.tmp_x .+= state.grad_f_x + state.tmp_x .*= (- state.gamma) + state.tmp_x .+= state.x + prox!(state.xbar, iter.g, state.tmp_x, state.gamma) + + # residuals + state.FPR_x .= state.xbar .- state.x + state.FPR_y .= state.ybar .- state.y + + return state, state +end + +default_stopping_criterion(tol, ::AdaptivePrimalDualIteration, state::AdaptivePrimalDualState) = + norm(state.FPR_x, Inf) + norm(state.FPR_y, Inf) <= tol +default_solution(::AdaptivePrimalDualIteration, state::AdaptivePrimalDualState) = (state.xbar, state.ybar) +default_display(it, ::AdaptivePrimalDualIteration, state::AdaptivePrimalDualState) = + @printf("%6d | %7.4e %7.4e\n", it, norm(state.FPR_x, Inf), norm(state.FPR_y, Inf)) + +""" + AdaptivePrimalDual(; ) + +Constructs the the adaptive three-term primal-dual algorithm (AdaPDM, see [1]). + +This algorithm solves convex optimization problems of the form + + minimize f(x) + g(x) + h(L x), + +where `f` is smooth, `g` and `h` are possibly nonsmooth, and `L` is a linear mapping. + +The returned object has type `IterativeAlgorithm{AdaptivePrimalDualIteration}`, +and can be called with the problem's arguments to trigger its solution. + +See also: [`AdaptivePrimalDualIteration`](@ref), [`IterativeAlgorithm`](@ref). + +# Arguments +- `maxit::Int=10_000`: maximum number of iteration +- `tol::1e-5`: tolerance for the default stopping criterion +- `stop::Function`: termination condition, `stop(::T, state)` should return `true` when to stop the iteration +- `solution::Function`: solution mapping, `solution(::T, state)` should return the identified solution +- `verbose::Bool=false`: whether the algorithm state should be displayed +- `freq::Int=100`: every how many iterations to display the algorithm state +- `display::Function`: display function, `display(::Int, ::T, state)` should display a summary of the iteration state +- `kwargs...`: additional keyword arguments to pass on to the `AdaptivePrimalDualIteration` constructor upon call + +# References +1. Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient", Mathematical Programming, vol. 213, pp. 433-471 (2025). +""" +AdaptivePrimalDual(; + maxit=10_000, + tol=1e-5, + stop=(iter, state) -> default_stopping_criterion(tol, iter, state), + solution=default_solution, + verbose=false, + freq=100, + display=default_display, + kwargs..., +) = IterativeAlgorithm( + AdaptivePrimalDualIteration; + maxit, + stop, + solution, + verbose, + freq, + display, + kwargs..., +) + + +# Aliases + +const AdaPDMState = AdaptivePrimalDualState +const AdaPDMIteration = AdaptivePrimalDualIteration +const AdaPDM = AdaptivePrimalDual \ No newline at end of file diff --git a/src/algorithms/adapgm.jl b/src/algorithms/adapgm.jl index 976758a8..9c6ffdab 100644 --- a/src/algorithms/adapgm.jl +++ b/src/algorithms/adapgm.jl @@ -83,8 +83,6 @@ function Base.iterate(iter::AdaptiveProximalGradientIteration) return state, state end -nan_to_zero(v) = ifelse(isnan(v), zero(v), v) - # stepsize rule as defined in Algorithm 2.1 of [1] function get_stepsize(gamma, x, grad_x, gamma_old, x_old, grad_x_old) LC = norm(grad_x - grad_x_old)^2 / norm(x - x_old)^2 |> nan_to_zero diff --git a/test/problems/test_elasticnet.jl b/test/problems/test_elasticnet.jl index 274d8ea4..fb1c2b42 100644 --- a/test/problems/test_elasticnet.jl +++ b/test/problems/test_elasticnet.jl @@ -24,7 +24,7 @@ using DifferentiationInterface: AutoZygote cost = ProximalAlgorithms.AutoDifferentiable(x -> (norm(A * x - b)^2) / 2, AutoZygote()) - L = opnorm(A)^2 + Lf = opnorm(A)^2 x_star = T[-0.6004983388704322, 0.0, 0.0, 0.195182724252491, 0.764119601328903] @@ -35,7 +35,7 @@ using DifferentiationInterface: AutoZygote x0 = zeros(T, n) x0_backup = copy(x0) solver = ProximalAlgorithms.DavisYin(tol = R(1e-6)) - xf_dys, it_dys = @inferred solver(x0 = x0, f = cost, g = reg1, h = reg2, Lf = L) + xf_dys, it_dys = @inferred solver(x0 = x0, f = cost, g = reg1, h = reg2, Lf = Lf) @test eltype(xf_dys) == T @test norm(xf_dys - x_star, Inf) <= 1e-3 @test it_dys <= 140 @@ -46,7 +46,7 @@ using DifferentiationInterface: AutoZygote x0 = randn(T, n) x0_backup = copy(x0) solver = ProximalAlgorithms.DavisYin(tol = R(1e-6)) - xf_dys, it_dys = @inferred solver(x0 = x0, f = cost, g = reg1, h = reg2, Lf = L) + xf_dys, it_dys = @inferred solver(x0 = x0, f = cost, g = reg1, h = reg2, Lf = Lf) @test eltype(xf_dys) == T @test norm(xf_dys - x_star, Inf) <= 1e-3 @test x0 == x0_backup @@ -105,4 +105,57 @@ using DifferentiationInterface: AutoZygote @test y0 == y0_backup end + + adapdm_test_params = [(1.0, 1e-6, 350), (2.0, 1e-6, 300), (1.0, 1e-3, 350), (1.0, 1.0, 700)] + + @testset "AdaPDM" for (t, epsil, maxit) in adapdm_test_params + + # with known initial iterates + x0 = zeros(T, n) + x0_backup = copy(x0) + y0 = zeros(T, m) + y0_backup = copy(y0) + + solver = ProximalAlgorithms.AdaptivePrimalDual(tol=R(1e-6)) + (x_adapdm, y_adapdm), it_adapdm = solver( + x0=x0, + y0=y0, + f=ProximalAlgorithms.AutoDifferentiable(reg2, AutoZygote()), + g=reg1, + h=loss, + L=A, + t=t, + epsil=epsil, + ) + @test eltype(x_adapdm) == T + @test eltype(y_adapdm) == T + @test norm(x_adapdm - x_star, Inf) <= 1e-4 + @test it_adapdm <= maxit + @test x0 == x0_backup + @test y0 == y0_backup + + # with random initial iterates + x0 = randn(T, n) + x0_backup = copy(x0) + y0 = randn(T, m) + y0_backup = copy(y0) + + solver = ProximalAlgorithms.AdaptivePrimalDual(tol=R(1e-6)) + (x_adapdm, y_adapdm), it_adapdm = solver( + x0=x0, + y0=y0, + f=ProximalAlgorithms.AutoDifferentiable(reg2, AutoZygote()), + g=reg1, + h=loss, + L=A, + t=t, + epsil=epsil, + ) + @test eltype(x_adapdm) == T + @test eltype(y_adapdm) == T + @test norm(x_adapdm - x_star, Inf) <= 1e-4 + @test it_adapdm <= maxit + @test x0 == x0_backup + @test y0 == y0_backup + end end diff --git a/test/problems/test_lasso_small.jl b/test/problems/test_lasso_small.jl index 09fb606b..c4e1f329 100644 --- a/test/problems/test_lasso_small.jl +++ b/test/problems/test_lasso_small.jl @@ -134,7 +134,7 @@ using ProximalAlgorithms: @test x0 == x0_backup end - @testset "AdaptiveProximalGradient (with Lf)" begin + @testset "AdaPGM (with Lf)" begin x0 = zeros(T, n) x0_backup = copy(x0) solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) @@ -145,7 +145,7 @@ using ProximalAlgorithms: @test x0 == x0_backup end - @testset "AdaptiveProximalGradient (w/o Lf)" begin + @testset "AdaPGM (w/o Lf)" begin x0 = zeros(T, n) x0_backup = copy(x0) solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) @@ -365,6 +365,45 @@ using ProximalAlgorithms: @test x0 == x0_backup end + @testset "AdaPDM" begin + x0 = zeros(T, n) + x0_backup = copy(x0) + solver = ProximalAlgorithms.AdaptivePrimalDual(tol = R(1e-6)) + (x_adapdm, y_adapdm), it_adapdm = @inferred solver( + x0 = x0, + y0 = zeros(T, n), + f = fA_autodiff, + g = g, + ) + @test eltype(x_adapdm) == T + @test eltype(y_adapdm) == T + @test norm(x_adapdm - x_star, Inf) <= 1e-4 + @test it_adapdm <= 80 + @test x0 == x0_backup + + solver = ProximalAlgorithms.AdaPDM(tol = R(1e-6)) + (x_adapdm, y_adapdm), it_adapdm = @inferred solver( + x0 = x0, + y0 = zeros(T, n), + f = fA_autodiff, + h = g, + ) + @test eltype(x_adapdm) == T + @test eltype(y_adapdm) == T + @test norm(x_adapdm - x_star, Inf) <= 1e-4 + @test it_adapdm <= 2700 + @test x0 == x0_backup + + solver = ProximalAlgorithms.AdaPDM(tol = R(1e-6)) + (x_adapdm, y_adapdm), it_adapdm = + @inferred solver(x0 = x0, y0 = zeros(T, m), h = f_prox, L = A, g = g) + @test eltype(x_adapdm) == T + @test eltype(y_adapdm) == T + @test norm(x_adapdm - x_star, Inf) <= 1e-4 + @test it_adapdm <= 550 + @test x0 == x0_backup + end + @testset "SFISTA" begin x0 = zeros(T, n) x0_backup = copy(x0) diff --git a/test/problems/test_lasso_small_strongly_convex.jl b/test/problems/test_lasso_small_strongly_convex.jl index 836639db..48bf920f 100644 --- a/test/problems/test_lasso_small_strongly_convex.jl +++ b/test/problems/test_lasso_small_strongly_convex.jl @@ -143,7 +143,7 @@ using ProximalAlgorithms @test x0 == x0_backup end - @testset "AdaptiveProximalGradient (with Lf)" begin + @testset "AdaPGM (with Lf)" begin solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) y, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf = Lf) @test eltype(y) == T @@ -152,7 +152,7 @@ using ProximalAlgorithms @test x0 == x0_backup end - @testset "AdaptiveProximalGradient (w/o Lf)" begin + @testset "AdaPGM (w/o Lf)" begin solver = ProximalAlgorithms.AdaptiveProximalGradient(tol = TOL) y, it = solver(x0 = x0, f = fA_autodiff, g = g) @test eltype(y) == T @@ -179,6 +179,38 @@ using ProximalAlgorithms @test x0 == x0_backup end + @testset "AdaPDM" begin + solver = ProximalAlgorithms.AdaptivePrimalDual(tol = TOL) + (v, vy), it = solver( + x0 = x0, + y0 = zeros(T, length(x0)), + f = fA_autodiff, + g = g, + ) + @test eltype(v) == T + @test norm(v - x_star, Inf) <= 5*TOL + @test it <= 40 + @test x0 == x0_backup + + (v, vy), it = solver( + x0 = x0, + y0 = zeros(T, length(x0)), + f = fA_autodiff, + h = g, + ) + @test eltype(v) == T + @test norm(v - x_star, Inf) <= 5*TOL + @test it <= 90 + @test x0 == x0_backup + + (v, vy), it = solver(x0 = x0, y0 = zeros(T, length(x0)), h = fA_prox, g = g) + @test eltype(v) == T + @test eltype(vy) == T + @test norm(v - x_star, Inf) <= 5*TOL + @test it <= 120 + @test x0 == x0_backup + end + @testset "ZeroFPR (fixed, nonmonotone)" begin solver = ProximalAlgorithms.ZeroFPR(tol = TOL, monotonicity = T(0.2)) y, it = solver(x0 = x0, f = fA_autodiff, g = g, Lf=Lf) diff --git a/test/problems/test_linear_programs.jl b/test/problems/test_linear_programs.jl index 54a908a3..6b67d38f 100644 --- a/test/problems/test_linear_programs.jl +++ b/test/problems/test_linear_programs.jl @@ -190,10 +190,26 @@ end @test it <= maxit - @assert norm(xf - x_star) <= 1e2 * tol + @test norm(xf - x_star) <= 1e2 * tol @test x0 == x0_backup + end + + @testset "AdaPDM" begin + + f = ProximalAlgorithms.AutoDifferentiable(x -> dot(c, x), AutoZygote()) + g = IndNonnegative() + x0 = zeros(T, n) + x0_backup = copy(x0) + + solver = ProximalAlgorithms.AdaPDM(tol = tol, maxit = maxit) + (v, vy), it = solver(x0 = x0, y0=zeros(T, n), f = f, g = g, h = IndAffine(A, b)) + + @test eltype(v) == T + @test it <= maxit + @test norm(v - x_star) <= 1e2 * tol + @test x0 == x0_backup end end diff --git a/test/problems/test_sparse_logistic_small.jl b/test/problems/test_sparse_logistic_small.jl index 16762fd2..3c562b6c 100644 --- a/test/problems/test_sparse_logistic_small.jl +++ b/test/problems/test_sparse_logistic_small.jl @@ -98,6 +98,25 @@ using LinearAlgebra @test x0 == x0_backup end + @testset "AdaPDM" begin + x0 = zeros(T, n) + x0_backup = copy(x0) + solver = ProximalAlgorithms.AdaPDM(tol = TOL) + (x, vy), it = solver(x0 = x0, y0=zeros(T,n), f = fA_autodiff, g = g) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it < 270 + @test x0 == x0_backup + + x0 = zeros(T, n) + x0_backup = copy(x0) + (x, vy), it = solver(x0 = x0, y0=zeros(T,n), f = fA_autodiff, h = g) + @test eltype(x) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it < 270 + @test x0 == x0_backup + end + @testset "ZeroFPR (adaptive step)" begin x0 = zeros(T, n) x0_backup = copy(x0) diff --git a/test/problems/test_verbose.jl b/test/problems/test_verbose.jl index 1112ef91..7e50fc12 100644 --- a/test/problems/test_verbose.jl +++ b/test/problems/test_verbose.jl @@ -178,4 +178,13 @@ using LinearAlgebra end + @testset "AdaPDM" begin + x0 = zeros(T, n) + solver = ProximalAlgorithms.AdaPDM(tol=TOL, verbose=true) + (x, y), it = solver(x0=x0, y0=zeros(T, n), g=fA_prox, h=g) + + @test eltype(x) == T + @test norm(x - x_star, Inf) <= 5*TOL + @test it < 250 + end end From 6877c1a97279fcda4f070e9c4f2eba9342db864b Mon Sep 17 00:00:00 2001 From: Alberto De Marchi Date: Sun, 15 Nov 2026 12:24:20 +0100 Subject: [PATCH 5/8] update docs with AdaPGM and AdaPDM --- docs/references.bib | 18 ++++++++++++++++-- docs/src/guide/implemented_algorithms.md | 6 ++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/references.bib b/docs/references.bib index d8c39873..b46124cd 100644 --- a/docs/references.bib +++ b/docs/references.bib @@ -189,7 +189,21 @@ @article{DeMarchi2022 @article{DeMarchi2024, title={An interior proximal gradient method for nonconvex optimization}, author={De Marchi, Alberto and Themelis, Andreas}, - journal={arXiv:2208.00799v2}, + journal={Open Journal of Mathematical Optimization}, + volume={5}, + eid = {3}, + pages = {1--22}, year={2024}, - url={https://doi.org/10.48550/arXiv.2208.00799} + url={https://doi.org/10.5802/ojmo.30} +} + +@article{Latafat2025, + title={Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient}, + author={Latafat, Puya and Themelis, Andreas and Stella, Lorenzo and Patrinos, Panagiotis}, + journal={Mathematical Programming}, + volume={213}, + number={1}, + pages={433--471}, + year={2025}, + url={https://doi.org/10.1007/s10107-024-02143-7} } \ No newline at end of file diff --git a/docs/src/guide/implemented_algorithms.md b/docs/src/guide/implemented_algorithms.md index a14fe6b9..5059ee8d 100644 --- a/docs/src/guide/implemented_algorithms.md +++ b/docs/src/guide/implemented_algorithms.md @@ -27,6 +27,7 @@ Algorithm | Assumptions | Oracle | Implementation | References Proximal gradient | ``f`` locally smooth | ``\nabla f``, ``\operatorname{prox}_{\gamma g}`` | [`ForwardBackward`](@ref) | [Lions1979](@cite), [DeMarchi2024](@cite) Douglas-Rachford | | ``\operatorname{prox}_{\gamma f}``, ``\operatorname{prox}_{\gamma g}`` | [`DouglasRachford`](@ref) | [Eckstein1992](@cite) Fast proximal gradient | ``f`` convex, smooth, ``g`` convex | ``\nabla f``, ``\operatorname{prox}_{\gamma g}`` | [`FastForwardBackward`](@ref) | [Tseng2008](@cite), [Beck2009](@cite) +AdaPGM | ``f`` convex, smooth, ``g`` convex | ``\nabla f``, ``\operatorname{prox}_{\gamma g}`` | [`AdaptiveProximalGradient`](@ref) | [Latafat2025](@cite) PANOC | ``f`` smooth | ``\nabla f``, ``\operatorname{prox}_{\gamma g}`` | [`PANOC`](@ref) | [Stella2017](@cite) ZeroFPR | ``f`` smooth | ``\nabla f``, ``\operatorname{prox}_{\gamma g}`` | [`ZeroFPR`](@ref) | [Themelis2018](@cite) Douglas-Rachford line-search | ``f`` smooth | ``\operatorname{prox}_{\gamma f}``, ``\operatorname{prox}_{\gamma g}`` | [`DRLS`](@ref) | [Themelis2022](@cite) @@ -39,6 +40,8 @@ ProximalAlgorithms.DouglasRachford ProximalAlgorithms.DouglasRachfordIteration ProximalAlgorithms.FastForwardBackward ProximalAlgorithms.FastForwardBackwardIteration +ProximalAlgorithms.AdaptiveProximalGradient +ProximalAlgorithms.AdaptiveProximalGradientIteration ProximalAlgorithms.PANOC ProximalAlgorithms.PANOCIteration ProximalAlgorithms.ZeroFPR @@ -74,6 +77,7 @@ Algorithm | Assumptions | Oracle | Implementation | References Chambolle-Pock | ``f\equiv 0``, ``g, h`` convex, ``L`` linear operator | ``\operatorname{prox}_{\gamma g}``, ``\operatorname{prox}_{\gamma h}``, ``L``, ``L^*`` | [`ChambollePock`](@ref) | [Chambolle2011](@cite) Vu-Condat | ``f`` convex and smooth, ``g, h`` convex, ``L`` linear operator | ``\nabla f``, ``\operatorname{prox}_{\gamma g}``, ``\operatorname{prox}_{\gamma h}``, ``L``, ``L^*`` | [`VuCondat`](@ref) | [Vu2013](@cite), [Condat2013](@cite) AFBA | ``f`` convex and smooth, ``g, h`` convex, ``L`` linear operator | ``\nabla f``, ``\operatorname{prox}_{\gamma g}``, ``\operatorname{prox}_{\gamma h}``, ``L``, ``L^*`` | [`AFBA`](@ref) | [Latafat2017](@cite) +AdaPDM | ``f`` convex and smooth, ``g, h`` convex, ``L`` linear operator | ``\nabla f``, ``\operatorname{prox}_{\gamma g}``, ``\operatorname{prox}_{\gamma h}``, ``L``, ``L^*`` | [`AdaptivePrimalDual`](@ref) | [Latafat2025](@cite) ```@docs ProximalAlgorithms.ChambollePock @@ -82,4 +86,6 @@ ProximalAlgorithms.VuCondat ProximalAlgorithms.VuCondatIteration ProximalAlgorithms.AFBA ProximalAlgorithms.AFBAIteration +ProximalAlgorithms.AdaptivePrimalDual +ProximalAlgorithms.AdaptivePrimalDualIteration ``` From 8ca4fd809e90509b061c404ed030653490bf7491 Mon Sep 17 00:00:00 2001 From: Alberto De Marchi Date: Mon, 16 Nov 2026 07:36:48 +0100 Subject: [PATCH 6/8] improve typing --- src/algorithms/adapdm.jl | 41 ++++++++++++++++---------------- test/problems/test_elasticnet.jl | 28 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/algorithms/adapdm.jl b/src/algorithms/adapdm.jl index 05a8c622..8f06b3a3 100644 --- a/src/algorithms/adapdm.jl +++ b/src/algorithms/adapdm.jl @@ -8,7 +8,7 @@ using ProximalCore: Zero, convex_conjugate using LinearAlgebra using Printf -# TODO implement AdaPDMplus +# TODO implement AdaPDMplus (Algorithm 3.2 of [1]) """ AdaptivePrimalDualIteration(; ) @@ -25,8 +25,8 @@ Points `x0` and `y0` are the initial primal and dual iterates, respectively. If unspecified, functions `f`, `g`, and `h` default to the identically zero function, and `L` defaults to the identity. -The iterator implements Algorithms 3.1 and 3.2 of [1], the latter being a linesearch -variant that does not require the linear operator norm. +The iterator implements Algorithm 3.1 of [1]. A linesearch variant that does not require +the linear operator norm is given by Algorithm 3.2 of [1], not yet implemented. See also: [`AdaptivePrimalDual`](@ref). @@ -40,22 +40,22 @@ See also: [`AdaptivePrimalDual`](@ref). - `Lopnorm=opnorm(L)`: operator norm of `L`. - `t=1`: positive algorithm parameter. - `epsil=1e-6`: positive algorithm parameter. -- `nu=1.2`: algorithm parameter such that `nu > 1 + epsil`. +- `nu=max(1.2,1+2*epsil)`: algorithm parameter such that `nu > 1 + epsil`. # References 1. Latafat, Themelis, Stella, Patrinos, "Adaptive proximal algorithms for convex optimization under local Lipschitz continuity of the gradient", Mathematical Programming, vol. 213, pp. 433-471 (2025). """ -Base.@kwdef struct AdaptivePrimalDualIteration{R,Tx,Ty,Tf,Tg,Th,TL,Tnormop} +Base.@kwdef struct AdaptivePrimalDualIteration{R<:Real,Tx,Ty,Tf,Tg,Th,TL,Tnormop} f::Tf = Zero() g::Tg = Zero() h::Th = Zero() L::TL = I x0::Tx y0::Ty - Lopnorm::Tnormop = LinearAlgebra.opnorm(L) # TODO not provided? then linesearch variant - t::R = 1.0 - epsil::R = 1e-6 - nu::R = max(1.2, 1+2*epsil) + Lopnorm::Tnormop = LinearAlgebra.opnorm(L) # TODO not provided? then linesearch variant AdaPDMplus + t::R = real(eltype(x0))(1) + epsil::R = real(eltype(x0))(1e-6) + nu::R = max(real(eltype(x0))(1.2), 1+2*epsil) end Base.IteratorSize(::Type{<:AdaptivePrimalDualIteration}) = Base.IsInfinite() @@ -78,20 +78,17 @@ Base.@kwdef mutable struct AdaptivePrimalDualState{R,Tx,Ty} end function Base.iterate(iter::AdaptivePrimalDualIteration) - @assert isreal(iter.Lopnorm) - @assert isreal(iter.t) - @assert isreal(iter.epsil) - @assert isreal(iter.nu) - @assert iter.Lopnorm > 0 - @assert iter.t > 0 - @assert iter.epsil > 0 - @assert iter.nu > 1 + iter.epsil + iter.Lopnorm > 0 || throw(ArgumentError("`Lopnorm` must be positive, got $iter.Lopnorm")) + iter.t > 0 || throw(ArgumentError("`t` must be positive, got $iter.t")) + iter.epsil > 0 || throw(ArgumentError("`epsil` must be positive, got $iter.epsil")) + iter.nu > 1 + iter.epsil || throw(ArgumentError("`nu` must satisfy nu > 1 + epsil, got nu = $iter.nu, epsil = $iter.epsil")) x = copy(iter.x0) y = copy(iter.y0) L_x = similar(y) mul!(L_x, iter.L, x) - gamma = 1/(2 * iter.nu * iter.t * iter.Lopnorm) + R = real(eltype(x)) + gamma = R(1/(2 * iter.nu * iter.t * iter.Lopnorm)) state = AdaptivePrimalDualState( x=x, @@ -128,6 +125,7 @@ end # stepsize rule as defined in Algorithm 3.1 of [1] function get_stepsizes(iter::AdaptivePrimalDualIteration, gamma, x, grad_x, gamma_old, x_old, grad_x_old) + R = real(eltype(x)) t = iter.t nu = iter.nu epsil = iter.epsil @@ -146,7 +144,7 @@ function get_stepsizes(iter::AdaptivePrimalDualIteration, gamma, x, grad_x, gamm cc2 = 1/(2*nu*t*Lopnorm) cc0 = 1-4*xi*(1+epsil)^2 cc3 = gamma*sqrt(cc0/(2*(1+epsil)*(sqrt(delta^2 + xi*cc0) + delta))) - gamma_new = min(cc1, cc2, cc3) + gamma_new = R(min(cc1, cc2, cc3)) return gamma_new, gamma end @@ -196,7 +194,7 @@ default_display(it, ::AdaptivePrimalDualIteration, state::AdaptivePrimalDualStat """ AdaptivePrimalDual(; ) -Constructs the the adaptive three-term primal-dual algorithm (AdaPDM, see [1]). +Constructs the adaptive linesearch-free three-term primal-dual algorithm (AdaPDM, see [1]). This algorithm solves convex optimization problems of the form @@ -206,12 +204,13 @@ where `f` is smooth, `g` and `h` are possibly nonsmooth, and `L` is a linear map The returned object has type `IterativeAlgorithm{AdaptivePrimalDualIteration}`, and can be called with the problem's arguments to trigger its solution. +The solution is returned as an (x, y) tuple. See also: [`AdaptivePrimalDualIteration`](@ref), [`IterativeAlgorithm`](@ref). # Arguments - `maxit::Int=10_000`: maximum number of iteration -- `tol::1e-5`: tolerance for the default stopping criterion +- `tol=1e-5`: tolerance for the default stopping criterion - `stop::Function`: termination condition, `stop(::T, state)` should return `true` when to stop the iteration - `solution::Function`: solution mapping, `solution(::T, state)` should return the identified solution - `verbose::Bool=false`: whether the algorithm state should be displayed diff --git a/test/problems/test_elasticnet.jl b/test/problems/test_elasticnet.jl index fb1c2b42..be0588f7 100644 --- a/test/problems/test_elasticnet.jl +++ b/test/problems/test_elasticnet.jl @@ -157,5 +157,33 @@ using DifferentiationInterface: AutoZygote @test it_adapdm <= maxit @test x0 == x0_backup @test y0 == y0_backup + + # with explicit Lopnorm::Float64 to test type stability + (x_adapdm, y_adapdm), it_adapdm = solver( + x0=x0, + y0=y0, + f=ProximalAlgorithms.AutoDifferentiable(reg2, AutoZygote()), + g=reg1, + h=loss, + L=A, + t=t, + epsil=epsil, + Lopnorm=Float64(2*opnorm(A)), + ) + @test eltype(x_adapdm) == T + @test eltype(y_adapdm) == T + @test norm(x_adapdm - x_star, Inf) <= 1e-4 + @test it_adapdm <= 2*maxit + @test x0 == x0_backup + @test y0 == y0_backup + + # input arguments + @test_throws MethodError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, epsil=0) + @test_throws MethodError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, t=0) + @test_throws MethodError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, nu=1) + @test_throws ArgumentError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, epsil=R(0)) + @test_throws ArgumentError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, t=R(0)) + @test_throws ArgumentError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, nu=R(1)) + @test_throws ArgumentError solver(x0=x0, y0=y0, f=cost, g=reg1, h=loss, L=A, Lopnorm=0) end end From 0317a11a413a8d9fba315e7b2541dd93840ca321 Mon Sep 17 00:00:00 2001 From: Alberto De Marchi Date: Mon, 16 Nov 2026 11:38:53 +0100 Subject: [PATCH 7/8] improve allocations --- src/algorithms/adapdm.jl | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/algorithms/adapdm.jl b/src/algorithms/adapdm.jl index 8f06b3a3..cfd19129 100644 --- a/src/algorithms/adapdm.jl +++ b/src/algorithms/adapdm.jl @@ -125,15 +125,20 @@ end # stepsize rule as defined in Algorithm 3.1 of [1] function get_stepsizes(iter::AdaptivePrimalDualIteration, gamma, x, grad_x, gamma_old, x_old, grad_x_old) - R = real(eltype(x)) + return get_stepsizes(iter, gamma, gamma_old, x-x_old, grad_x-grad_x_old) +end + +function get_stepsizes(iter::AdaptivePrimalDualIteration, gamma, gamma_old, delta_x, delta_grad_x) + R = real(eltype(delta_x)) t = iter.t nu = iter.nu epsil = iter.epsil Lopnorm = iter.Lopnorm # ℓ_k and c_k from (1.3) of [1] - l = real(dot(grad_x - grad_x_old, x - x_old)) / norm(x - x_old)^2 |> nan_to_zero - lc = norm(grad_x - grad_x_old)^2 / norm(x - x_old)^2 |> nan_to_zero + dx2 = norm(delta_x)^2 + l = real(dot(delta_grad_x, delta_x)) / dx2 |> nan_to_zero + lc = norm(delta_grad_x)^2 / dx2 |> nan_to_zero # δ_k and ξ_k delta = gamma * (gamma*lc - l) |> nan_to_zero @@ -158,7 +163,9 @@ function Base.iterate( mul!(state.L_x, iter.L, state.xbar) # stepsizes - state.gamma, state.gamma_old = get_stepsizes(iter, state.gamma, state.xbar, state.grad_f_x, state.gamma_old, state.x, state.grad_f_xold) + state.tmp_x .= state.xbar .- state.x # temporary + state.FPR_x .= state.grad_f_x .- state.grad_f_xold # temporary + state.gamma, state.gamma_old = get_stepsizes(iter, state.gamma, state.gamma_old, state.tmp_x, state.FPR_x) sigma = state.gamma * iter.t^2 state.x .= state.xbar @@ -166,7 +173,7 @@ function Base.iterate( # y-update rho = state.gamma / state.gamma_old - state.tmp_y .= (1+rho) .* state.L_x - rho .* state.L_xold + state.tmp_y .= (1+rho) .* state.L_x .- rho .* state.L_xold state.tmp_y .*= sigma state.tmp_y .+= state.y prox!(state.ybar, convex_conjugate(iter.h), state.tmp_y, sigma) From 7672a9f0a81201ecf4fac4642eb619072131ca2a Mon Sep 17 00:00:00 2001 From: Alberto De Marchi Date: Wed, 18 Nov 2026 10:43:31 +0100 Subject: [PATCH 8/8] adapdm: improve residuals and termination --- src/algorithms/adapdm.jl | 22 +++++++---- test/problems/test_elasticnet.jl | 37 ++++++++++--------- test/problems/test_lasso_small.jl | 11 +++--- .../test_lasso_small_strongly_convex.jl | 6 +-- test/problems/test_linear_programs.jl | 2 +- test/problems/test_sparse_logistic_small.jl | 4 +- test/problems/test_verbose.jl | 7 ++++ 7 files changed, 52 insertions(+), 37 deletions(-) diff --git a/src/algorithms/adapdm.jl b/src/algorithms/adapdm.jl index cfd19129..a3115c86 100644 --- a/src/algorithms/adapdm.jl +++ b/src/algorithms/adapdm.jl @@ -10,6 +10,11 @@ using Printf # TODO implement AdaPDMplus (Algorithm 3.2 of [1]) +# TODO Default Lopnorm = opnorm(L) +# For a dense L this is a full SVD, which can be expensive at scale. It also fails or is +# impractical for sparse or matrix-free operators. Document that users can pass Lopnorm +# themselves, or use a power-iteration estimate as the default (with a safety factor, since +# an underestimate breaks the stepsize bound). """ AdaptivePrimalDualIteration(; ) @@ -168,6 +173,10 @@ function Base.iterate( state.gamma, state.gamma_old = get_stepsizes(iter, state.gamma, state.gamma_old, state.tmp_x, state.FPR_x) sigma = state.gamma * iter.t^2 + # x-residual + # (tmp_x = xbar - x, FPR_x = grad_x - grad_xold at this point) + state.FPR_x .-= state.tmp_x ./ state.gamma_old + state.x .= state.xbar state.y .= state.ybar @@ -178,6 +187,9 @@ function Base.iterate( state.tmp_y .+= state.y prox!(state.ybar, convex_conjugate(iter.h), state.tmp_y, sigma) + # y-residual + state.FPR_y .= (state.tmp_y .- state.ybar) ./ sigma .- state.L_x + # x-update mul!(state.tmp_x, iter.L', state.ybar) state.tmp_x .+= state.grad_f_x @@ -185,18 +197,14 @@ function Base.iterate( state.tmp_x .+= state.x prox!(state.xbar, iter.g, state.tmp_x, state.gamma) - # residuals - state.FPR_x .= state.xbar .- state.x - state.FPR_y .= state.ybar .- state.y - return state, state end default_stopping_criterion(tol, ::AdaptivePrimalDualIteration, state::AdaptivePrimalDualState) = - norm(state.FPR_x, Inf) + norm(state.FPR_y, Inf) <= tol -default_solution(::AdaptivePrimalDualIteration, state::AdaptivePrimalDualState) = (state.xbar, state.ybar) + sqrt(norm(state.FPR_x)^2 + norm(state.FPR_y)^2) <= tol +default_solution(::AdaptivePrimalDualIteration, state::AdaptivePrimalDualState) = (state.x, state.ybar) default_display(it, ::AdaptivePrimalDualIteration, state::AdaptivePrimalDualState) = - @printf("%6d | %7.4e %7.4e\n", it, norm(state.FPR_x, Inf), norm(state.FPR_y, Inf)) + @printf("%6d | %7.4e %7.4e\n", it, norm(state.FPR_x), norm(state.FPR_y)) """ AdaptivePrimalDual(; ) diff --git a/test/problems/test_elasticnet.jl b/test/problems/test_elasticnet.jl index be0588f7..3c3e34d4 100644 --- a/test/problems/test_elasticnet.jl +++ b/test/problems/test_elasticnet.jl @@ -106,9 +106,10 @@ using DifferentiationInterface: AutoZygote end - adapdm_test_params = [(1.0, 1e-6, 350), (2.0, 1e-6, 300), (1.0, 1e-3, 350), (1.0, 1.0, 700)] + adapdm_test_params = [(1.0, 1e-6, 450), (2.0, 1e-6, 450), (1.0, 1e-3, 450), (1.0, 1.0, 1000)] @testset "AdaPDM" for (t, epsil, maxit) in adapdm_test_params + tmptol = T in [Float32, ComplexF32] ? R(1e-5) : R(1e-7) # with known initial iterates x0 = zeros(T, n) @@ -116,8 +117,8 @@ using DifferentiationInterface: AutoZygote y0 = zeros(T, m) y0_backup = copy(y0) - solver = ProximalAlgorithms.AdaptivePrimalDual(tol=R(1e-6)) - (x_adapdm, y_adapdm), it_adapdm = solver( + solver = ProximalAlgorithms.AdaptivePrimalDual(tol=tmptol) + (x, y), it = solver( x0=x0, y0=y0, f=ProximalAlgorithms.AutoDifferentiable(reg2, AutoZygote()), @@ -127,10 +128,10 @@ using DifferentiationInterface: AutoZygote t=t, epsil=epsil, ) - @test eltype(x_adapdm) == T - @test eltype(y_adapdm) == T - @test norm(x_adapdm - x_star, Inf) <= 1e-4 - @test it_adapdm <= maxit + @test eltype(x) == T + @test eltype(y) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it <= maxit @test x0 == x0_backup @test y0 == y0_backup @@ -140,8 +141,8 @@ using DifferentiationInterface: AutoZygote y0 = randn(T, m) y0_backup = copy(y0) - solver = ProximalAlgorithms.AdaptivePrimalDual(tol=R(1e-6)) - (x_adapdm, y_adapdm), it_adapdm = solver( + solver = ProximalAlgorithms.AdaptivePrimalDual(tol=tmptol) + (x, y), it = solver( x0=x0, y0=y0, f=ProximalAlgorithms.AutoDifferentiable(reg2, AutoZygote()), @@ -151,15 +152,15 @@ using DifferentiationInterface: AutoZygote t=t, epsil=epsil, ) - @test eltype(x_adapdm) == T - @test eltype(y_adapdm) == T - @test norm(x_adapdm - x_star, Inf) <= 1e-4 - @test it_adapdm <= maxit + @test eltype(x) == T + @test eltype(y) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it <= maxit @test x0 == x0_backup @test y0 == y0_backup # with explicit Lopnorm::Float64 to test type stability - (x_adapdm, y_adapdm), it_adapdm = solver( + (x, y), it = solver( x0=x0, y0=y0, f=ProximalAlgorithms.AutoDifferentiable(reg2, AutoZygote()), @@ -170,10 +171,10 @@ using DifferentiationInterface: AutoZygote epsil=epsil, Lopnorm=Float64(2*opnorm(A)), ) - @test eltype(x_adapdm) == T - @test eltype(y_adapdm) == T - @test norm(x_adapdm - x_star, Inf) <= 1e-4 - @test it_adapdm <= 2*maxit + @test eltype(x) == T + @test eltype(y) == T + @test norm(x - x_star, Inf) <= 1e-4 + @test it <= 2*maxit @test x0 == x0_backup @test y0 == y0_backup diff --git a/test/problems/test_lasso_small.jl b/test/problems/test_lasso_small.jl index c4e1f329..2cdaa2fa 100644 --- a/test/problems/test_lasso_small.jl +++ b/test/problems/test_lasso_small.jl @@ -368,7 +368,7 @@ using ProximalAlgorithms: @testset "AdaPDM" begin x0 = zeros(T, n) x0_backup = copy(x0) - solver = ProximalAlgorithms.AdaptivePrimalDual(tol = R(1e-6)) + solver = ProximalAlgorithms.AdaptivePrimalDual(tol = TOL) (x_adapdm, y_adapdm), it_adapdm = @inferred solver( x0 = x0, y0 = zeros(T, n), @@ -378,10 +378,10 @@ using ProximalAlgorithms: @test eltype(x_adapdm) == T @test eltype(y_adapdm) == T @test norm(x_adapdm - x_star, Inf) <= 1e-4 - @test it_adapdm <= 80 + @test it_adapdm <= 130 @test x0 == x0_backup - solver = ProximalAlgorithms.AdaPDM(tol = R(1e-6)) + solver = ProximalAlgorithms.AdaPDM(tol = TOL) (x_adapdm, y_adapdm), it_adapdm = @inferred solver( x0 = x0, y0 = zeros(T, n), @@ -391,16 +391,15 @@ using ProximalAlgorithms: @test eltype(x_adapdm) == T @test eltype(y_adapdm) == T @test norm(x_adapdm - x_star, Inf) <= 1e-4 - @test it_adapdm <= 2700 + @test it_adapdm <= 3900 @test x0 == x0_backup - solver = ProximalAlgorithms.AdaPDM(tol = R(1e-6)) (x_adapdm, y_adapdm), it_adapdm = @inferred solver(x0 = x0, y0 = zeros(T, m), h = f_prox, L = A, g = g) @test eltype(x_adapdm) == T @test eltype(y_adapdm) == T @test norm(x_adapdm - x_star, Inf) <= 1e-4 - @test it_adapdm <= 550 + @test it_adapdm <= 700 @test x0 == x0_backup end diff --git a/test/problems/test_lasso_small_strongly_convex.jl b/test/problems/test_lasso_small_strongly_convex.jl index 48bf920f..32229624 100644 --- a/test/problems/test_lasso_small_strongly_convex.jl +++ b/test/problems/test_lasso_small_strongly_convex.jl @@ -189,7 +189,7 @@ using ProximalAlgorithms ) @test eltype(v) == T @test norm(v - x_star, Inf) <= 5*TOL - @test it <= 40 + @test it <= 50 @test x0 == x0_backup (v, vy), it = solver( @@ -200,14 +200,14 @@ using ProximalAlgorithms ) @test eltype(v) == T @test norm(v - x_star, Inf) <= 5*TOL - @test it <= 90 + @test it <= 100 @test x0 == x0_backup (v, vy), it = solver(x0 = x0, y0 = zeros(T, length(x0)), h = fA_prox, g = g) @test eltype(v) == T @test eltype(vy) == T @test norm(v - x_star, Inf) <= 5*TOL - @test it <= 120 + @test it <= 140 @test x0 == x0_backup end diff --git a/test/problems/test_linear_programs.jl b/test/problems/test_linear_programs.jl index 6b67d38f..400f11ba 100644 --- a/test/problems/test_linear_programs.jl +++ b/test/problems/test_linear_programs.jl @@ -207,7 +207,7 @@ end (v, vy), it = solver(x0 = x0, y0=zeros(T, n), f = f, g = g, h = IndAffine(A, b)) @test eltype(v) == T - @test it <= maxit + @test it <= 25_000 @test norm(v - x_star) <= 1e2 * tol @test x0 == x0_backup end diff --git a/test/problems/test_sparse_logistic_small.jl b/test/problems/test_sparse_logistic_small.jl index 3c562b6c..78f8f8a3 100644 --- a/test/problems/test_sparse_logistic_small.jl +++ b/test/problems/test_sparse_logistic_small.jl @@ -105,7 +105,7 @@ using LinearAlgebra (x, vy), it = solver(x0 = x0, y0=zeros(T,n), f = fA_autodiff, g = g) @test eltype(x) == T @test norm(x - x_star, Inf) <= 1e-4 - @test it < 270 + @test it < 280 @test x0 == x0_backup x0 = zeros(T, n) @@ -113,7 +113,7 @@ using LinearAlgebra (x, vy), it = solver(x0 = x0, y0=zeros(T,n), f = fA_autodiff, h = g) @test eltype(x) == T @test norm(x - x_star, Inf) <= 1e-4 - @test it < 270 + @test it < 280 @test x0 == x0_backup end diff --git a/test/problems/test_verbose.jl b/test/problems/test_verbose.jl index 7e50fc12..25688f02 100644 --- a/test/problems/test_verbose.jl +++ b/test/problems/test_verbose.jl @@ -186,5 +186,12 @@ using LinearAlgebra @test eltype(x) == T @test norm(x - x_star, Inf) <= 5*TOL @test it < 250 + + solver = ProximalAlgorithms.AdaPDM(tol=TOL, verbose=true, freq=42) + (x, y), it = solver(x0=x0, y0=zeros(T, n), g=fA_prox, h=g) + + @test eltype(x) == T + @test norm(x - x_star, Inf) <= 5*TOL + @test it < 250 end end