diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml new file mode 100644 index 0000000..131ad25 --- /dev/null +++ b/.github/workflows/draft-pdf.yml @@ -0,0 +1,27 @@ +name: Build JOSS paper draft PDF +on: + push: + branches: + - joss-paper + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + # This should be the path to the paper within your repo. + paper-path: paper/paper.md + - name: Upload pdf artifact + uses: actions/upload-artifact@v4 + with: + name: paper + # This is the output path where Pandoc will write the compiled + # PDF. Note, this should be the same directory as the input + # paper.md + path: paper/paper.pdf \ No newline at end of file diff --git a/paper/examples/Project.toml b/paper/examples/Project.toml new file mode 100644 index 0000000..de7b404 --- /dev/null +++ b/paper/examples/Project.toml @@ -0,0 +1,14 @@ +[deps] +DSP = "717857b8-e6f2-59f4-9121-6e50c889abd2" +DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" +FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" +LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +ProximalAlgorithms = "140ffc9f-1907-541a-a177-7475e0a401e9" +ProximalCore = "dc4f5ac2-75d1-4f31-931e-60435d74994b" +ProximalOperators = "a725b495-10eb-56fe-b38b-717eba820537" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" diff --git a/paper/examples/comparison_linespectraestimation.jl b/paper/examples/comparison_linespectraestimation.jl new file mode 100644 index 0000000..b2051bb --- /dev/null +++ b/paper/examples/comparison_linespectraestimation.jl @@ -0,0 +1,204 @@ +# this script for comparing different solvers is inspired by +# - https://github.com/JuliaFirstOrder/StructuredOptimization.jl/blob/master/demos/LineSpectraEstimation.ipynb +# - https://github.com/JuliaSmoothOptimizers/RegularizedOptimization.jl/blob/paper/paper/examples/Benchmark.jl +# - https://juliafirstorder.github.io/ProximalAlgorithms.jl/stable/guide/custom_objectives/#Example:-counting-operations + +using Random, SparseArrays, LinearAlgebra +using DSP, Statistics, FFTW +using Zygote +using DifferentiationInterface: AutoZygote +using ProximalCore, ProximalOperators, ProximalAlgorithms +using PrettyTables, LaTeXStrings + +#-------------------------------------------------------------- +# benchmark configuration +CFG = Dict( + "seed" => 1234, + "tol" => 1e-6, + "max_iter" => 1_000_000, + "verbose" => false, + "solvers" => [:ForwardBackward, :FastForwardBackward, :ZeroFPR, :PANOC, :PANOCplus], + "regularizers" => [:NormL1, :NormLinf, :NormL0, :IndBallL0], + "num_samples" => 20, + "print_table" => true, +) + +#-------------------------------------------------------------- +# counters +mutable struct Counting{T} + f::T + eval_count::Int + grad_count::Int + prox_count::Int +end + +Counting(f::T) where {T} = Counting{T}(f, 0, 0, 0) + +function (f::Counting)(x) + f.eval_count += 1 + return f.f(x) +end + +function ProximalAlgorithms.value_and_gradient(f::Counting, x) + f.eval_count += 1 + f.grad_count += 1 + return ProximalAlgorithms.value_and_gradient(f.f, x) +end + +function ProximalCore.prox!(y, f::Counting, x, gamma) + f.prox_count += 1 + return ProximalCore.prox!(y, f.f, x, gamma) +end + +#-------------------------------------------------------------- +# problem generation +function make_loss_fun(cfg) + Random.seed!(cfg["seed"]) # For reproducibility + + Fs = 16e3 #sampling frequency + Nt = 2^7 #time samples + y0 = zeros(Nt) + SNR = 20 + S = 4 #super-resolution factor + f_s = range(0, stop=Fs/2; length=div(S*Nt, 2)+1) # super resolution frequency axis + f = range(0, stop=Fs/2; length=div(Nt, 2)+1) # frequency axis + t = 0:(1/Fs):((Nt-1)/Fs) # time axis + N = 14 # number of sinusoids + Nf = div(S*Nt, 4)+1 + p = randperm(Nf)[1:N] + fk = f_s[p] # pick sinusoids frequencies + A = 0.1 .* randn(N) .+ 0.7 # amplitude + for i in eachindex(fk) + y0 .+= A[i] .* sin.(2*pi*fk[i] .* t) # mixture of sinusoids + end + y = y0 .+ 10^(-SNR/10)*sqrt(var(y0))*randn(length(y0)); # add noise + + x0 = randn(ComplexF64, S*Nt) + f = ProximalAlgorithms.AutoDifferentiable(x -> 0.5*norm(ifft(x)[1:Nt]-complex(y), 2)^2, AutoZygote()) + return f, x0 +end + +function run_solver(f, g, x0, solversym, cfg) + maxit = cfg["max_iter"] + tol = cfg["tol"] + verbose = cfg["verbose"] + solverbuilder = ProximalAlgorithms.eval(solversym) + solver = solverbuilder(maxit=maxit, tol=tol, verbose=verbose) + + fc = Counting(f) + gc = Counting(g) + + runtime = @elapsed x, iters = solver(f=fc, g=gc, x0=x0) + status = iters < maxit ? "first_order" : "max_iter" + objx = f(x)+g(x) + return ( + name=String(solversym), + objective=objx, + status=status, + runtime=runtime, + iters=iters, + fevals=fc.eval_count, + gradevals=fc.grad_count, + proxevals=gc.prox_count + ) +end + +function run_solver_mult(f, g, x0, solversym, cfg) + nsamples = cfg["num_samples"] + nx = length(x0) + objx = zeros(nsamples) + status = zeros(nsamples) + runtime = zeros(nsamples) + iters = zeros(nsamples) + fevals = zeros(nsamples) + gradevals = zeros(nsamples) + proxevals = zeros(nsamples) + for i = 1:nsamples + # random initial guess + seed = cfg["seed"] + i - 1 + Random.seed!(seed) # For reproducibility + x0 .= randn(ComplexF64, nx) + # call solver + res = run_solver(f, g, x0, solversym, cfg) + # store results + objx[i] = res.objective + status[i] = res.status == "first_order" ? 1 : 0 + runtime[i] = res.runtime + iters[i] = res.iters + fevals[i] = res.fevals + gradevals[i] = res.gradevals + proxevals[i] = res.proxevals + end + nsolved = Int(sum(status)) + return ( + name=String(solversym), + objective=median(objx), + status="$(nsolved)/$(nsamples)", + runtime=median(runtime), + iters=median(iters), + fevals=median(fevals), + gradevals=median(gradevals), + proxevals=median(proxevals) + ) +end + +function bench_regularizer(f, g, x0, cfg) + results = NamedTuple[] + for solversym in cfg["solvers"] + push!(results, run_solver_mult(f, g, x0, solversym, cfg)) + end + + restable = [ + (; name=m.name, + status=string(m.status), + runtime=round(m.runtime, digits=4), + gradevals=m.gradevals, + proxevals=m.proxevals, + objx=round(m.objective, digits=4)) + for m in results + ] + + return restable +end + +# main +function main(; cfg=CFG, latex_out=true) + f, x0 = make_loss_fun(cfg) + + for regsym in cfg["regularizers"] + regstr = String(regsym) + regbuilder = ProximalOperators.eval(regsym) + if regbuilder == IndBallL0 + λ = 28 # 2*N in LineSpectraEstimation.ipynb + else + λ = 1 + end + g = regbuilder(λ) + + restable = bench_regularizer(f, g, x0, cfg) + + if cfg["print_table"] + println("\n=== Results with "*regstr*" ===") + for row in restable + println(row) + end + end + + # save as latex format + if latex_out + table_str = pretty_table(String, + restable; + backend=:latex, + column_labels=["Method", "Success rate", L"$t$ ($s$)", L"$\#\nabla f$", L"$\#prox$", "Objective"], + style=LatexTableStyle(column_label=String[]), + table_format=latex_table_format__booktabs + ) + + open("table_"*regstr*".tex", "w") do io + write(io, table_str) + end + end + end + + return nothing +end \ No newline at end of file diff --git a/paper/examples/example.jl b/paper/examples/example.jl new file mode 100644 index 0000000..0c16bc3 --- /dev/null +++ b/paper/examples/example.jl @@ -0,0 +1,21 @@ +using ProximalOperators, ProximalAlgorithms + +struct Rosenbrock{T} + a::T +end +(f::Rosenbrock)(x) = (f.a-x[1])^2 + 100*(x[2]-x[1]^2)^2 +function ProximalAlgorithms.value_and_gradient(f::Rosenbrock, x) + return f(x), [2*(x[1]-f.a) + 400*(x[1]^2-x[2])*x[1], 200*(x[2]-x[1]^2)] +end + +f = Rosenbrock(1.0) +g = NormL1(2.0) + +solver = ProximalAlgorithms.ForwardBackward(tol = 1e-6, verbose = true) +x, iters = solver(x0 = ones(2), f = f, g = g) + +using Zygote +using DifferentiationInterface: AutoZygote + +f_auto = ProximalAlgorithms.AutoDifferentiable(x -> f(x), AutoZygote()) +x, iters = solver(x0 = ones(2), f = f_auto, g = g) \ No newline at end of file diff --git a/paper/examples/table_IndBallL0.tex b/paper/examples/table_IndBallL0.tex new file mode 100644 index 0000000..6258d01 --- /dev/null +++ b/paper/examples/table_IndBallL0.tex @@ -0,0 +1,11 @@ +\begin{tabular}{rrrrrr} + \toprule + \textbf{Method} & \textbf{Success rate} & \textbf{$t$ ($s$)} & \textbf{$\#\nabla f$} & \textbf{$\#prox$} & \textbf{Objective} \\ + \midrule + ForwardBackward & 20/20 & 0.5451 & 7509.5 & 7508.5 & 19.4546 \\ + FastForwardBackward & 20/20 & 0.1482 & 3066.0 & 1533.0 & 23.8179 \\ + ZeroFPR & 20/20 & 0.0374 & 521.5 & 520.5 & 18.7917 \\ + PANOC & 20/20 & 0.0253 & 523.0 & 269.5 & 11.1509 \\ + PANOCplus & 20/20 & 0.0272 & 517.0 & 258.0 & 11.1509 \\ + \bottomrule +\end{tabular} diff --git a/paper/examples/table_NormL0.tex b/paper/examples/table_NormL0.tex new file mode 100644 index 0000000..5b9c506 --- /dev/null +++ b/paper/examples/table_NormL0.tex @@ -0,0 +1,11 @@ +\begin{tabular}{rrrrrr} + \toprule + \textbf{Method} & \textbf{Success rate} & \textbf{$t$ ($s$)} & \textbf{$\#\nabla f$} & \textbf{$\#prox$} & \textbf{Objective} \\ + \midrule + ForwardBackward & 20/20 & 0.5405 & 17898.0 & 17897.0 & 28.0178 \\ + FastForwardBackward & 20/20 & 0.0791 & 2704.0 & 1352.0 & 28.0178 \\ + ZeroFPR & 20/20 & 0.0137 & 414.0 & 413.0 & 31.0171 \\ + PANOC & 20/20 & 0.0193 & 550.0 & 313.0 & 28.0178 \\ + PANOCplus & 20/20 & 0.0202 & 620.0 & 309.5 & 28.0178 \\ + \bottomrule +\end{tabular} diff --git a/paper/examples/table_NormL1.tex b/paper/examples/table_NormL1.tex new file mode 100644 index 0000000..43ba8f0 --- /dev/null +++ b/paper/examples/table_NormL1.tex @@ -0,0 +1,11 @@ +\begin{tabular}{rrrrrr} + \toprule + \textbf{Method} & \textbf{Success rate} & \textbf{$t$ ($s$)} & \textbf{$\#\nabla f$} & \textbf{$\#prox$} & \textbf{Objective} \\ + \midrule + ForwardBackward & 20/20 & 0.0001 & 3.0 & 2.0 & 231.763 \\ + FastForwardBackward & 20/20 & 0.0001 & 4.0 & 2.0 & 231.763 \\ + ZeroFPR & 20/20 & 0.0001 & 4.0 & 3.0 & 231.763 \\ + PANOC & 20/20 & 0.0001 & 4.0 & 2.0 & 231.763 \\ + PANOCplus & 20/20 & 0.0001 & 5.0 & 2.0 & 231.763 \\ + \bottomrule +\end{tabular} diff --git a/paper/examples/table_NormLinf.tex b/paper/examples/table_NormLinf.tex new file mode 100644 index 0000000..951f470 --- /dev/null +++ b/paper/examples/table_NormLinf.tex @@ -0,0 +1,11 @@ +\begin{tabular}{rrrrrr} + \toprule + \textbf{Method} & \textbf{Success rate} & \textbf{$t$ ($s$)} & \textbf{$\#\nabla f$} & \textbf{$\#prox$} & \textbf{Objective} \\ + \midrule + ForwardBackward & 20/20 & 0.2252 & 5883.5 & 5882.5 & 40.742 \\ + FastForwardBackward & 20/20 & 0.0328 & 1032.0 & 516.0 & 40.742 \\ + ZeroFPR & 20/20 & 0.0186 & 467.0 & 466.0 & 40.742 \\ + PANOC & 20/20 & 0.0281 & 811.0 & 433.0 & 40.742 \\ + PANOCplus & 20/20 & 0.0333 & 851.0 & 425.0 & 40.742 \\ + \bottomrule +\end{tabular} diff --git a/paper/paper.bib b/paper/paper.bib new file mode 100644 index 0000000..b0e2bd9 --- /dev/null +++ b/paper/paper.bib @@ -0,0 +1,318 @@ +@article{lions1979, + title={Splitting algorithms for the sum of two nonlinear operators}, + author={Lions, Pierre-Louis and Mercier, Bertrand}, + journal={SIAM Journal on Numerical Analysis}, + volume={16}, + number={6}, + pages={964--979}, + year={1979}, + doi={10.1137/0716071}, +} + +@article{beck2009, + title={A fast iterative shrinkage-thresholding algorithm for linear inverse problems}, + author={Beck, Amir and Teboulle, Marc}, + journal={SIAM Journal on Imaging Sciences}, + volume={2}, + number={1}, + pages={183--202}, + year={2009}, + doi={10.1137/080716542}, +} + +@article{eckstein1992, + title={On the {Douglas}--{Rachford} splitting method and the proximal point algorithm for maximal monotone operators}, + author={Eckstein, Jonathan and Bertsekas, Dimitri P}, + journal={Mathematical Programming}, + volume={55}, + number={1}, + pages={293--318}, + year={1992}, + doi={10.1007/BF01581204}, +} + +@inproceedings{stella2017, + title={A simple and efficient algorithm for nonlinear model predictive control}, + author={Stella, Lorenzo and Themelis, Andreas and Sopasakis, Pantelis and Patrinos, Panagiotis}, + booktitle={56th IEEE Conference on Decision and Control (CDC)}, + year={2017}, + pages={1939--1944}, + doi={10.1109/CDC.2017.8263933}, +} + +@article{demarchi2022, + title={Proximal gradient algorithms under local {Lipschitz} gradient continuity}, + author={De Marchi, Alberto and Themelis, Andreas}, + journal={Journal of Optimization Theory and Applications}, + volume={194}, + number={3}, + pages={771--794}, + year={2022}, + doi={10.1007/s10957-022-02048-5}, +} + +@article{themelis2018, + title={Forward-backward envelope for the sum of two nonconvex functions: Further properties and nonmonotone line-search algorithms}, + author={Themelis, Andreas and Stella, Lorenzo and Patrinos, Panagiotis}, + journal={SIAM Journal on Optimization}, + volume={28}, + number={3}, + pages={2274--2303}, + year={2018}, + doi={10.1137/16M1080240}, +} + +@article{themelis2022, + title={Douglas--{Rachford} splitting and {ADMM} for nonconvex optimization: Accelerated and {Newton}-type linesearch algorithms}, + author={Themelis, Andreas and Stella, Lorenzo and Patrinos, Panagiotis}, + journal={Computational Optimization and Applications}, + volume={82}, + number={2}, + pages={395--440}, + year={2022}, + doi={10.1007/s10589-022-00366-y}, +} + +@article{davis2017, + title={A three-operator splitting scheme and its optimization applications}, + author={Davis, Damek and Yin, Wotao}, + journal={Set-Valued and Variational Analysis}, + volume={25}, + number={4}, + pages={829--858}, + year={2017}, + doi={10.1007/s11228-017-0421-z}, +} + +@article{chambolle2011, + title={A first-order primal-dual algorithm for convex problems with applications to imaging}, + author={Chambolle, Antonin and Pock, Thomas}, + journal={Journal of Mathematical Imaging and Vision}, + volume={40}, + number={1}, + pages={120--145}, + year={2011}, + doi={10.1007/s10851-010-0251-1}, +} + +@article{vu2013, + title={A splitting algorithm for dual monotone inclusions involving cocoercive operators}, + author={V{\~u}, Bang C{\^o}ng}, + journal={Advances in Computational Mathematics}, + volume={38}, + number={3}, + pages={667--681}, + year={2013}, + doi={10.1007/s10444-011-9254-8}, +} + +@article{condat2013, + title={A primal-dual splitting method for convex optimization involving {Lipschitzian}, proximable and linear composite terms}, + author={Condat, Laurent}, + journal={Journal of Optimization Theory and Applications}, + volume={158}, + number={2}, + pages={460--479}, + year={2013}, + doi={10.1007/s10957-012-0245-9}, +} + +@article{latafat2017, + title={Asymmetric forward-backward-adjoint splitting for solving monotone inclusions involving three operators}, + author={Latafat, Puya and Patrinos, Panagiotis}, + journal={Computational Optimization and Applications}, + volume={68}, + number={1}, + pages={57--93}, + year={2017}, + doi={10.1007/s10589-017-9909-6}, +} + +@article{latafat2025, + title={Adaptive proximal algorithms for convex optimization under local {Lipschitz} continuity of the gradient}, + author={Latafat, Puya and Themelis, Andreas and Stella, Lorenzo and Patrinos, Panagiotis}, + journal={Mathematical Programming}, + volume={213}, + number={1}, + pages={433--471}, + year={2025}, + doi={10.1007/s10107-024-02143-7}, +} + +@article{gollier2026, + title={{RegularizedOptimization.jl}: A {Julia} framework for regularized and nonsmooth optimization}, + author={Gollier, Maxence and Habiboullah, Mohamed Laghdaf and Leconte, Geoffroy and Baraldi, Robert and De~Marchi, Alberto and Orban, Dominique and Diouane, Youssef}, + journal={Journal of Open Source Software}, + volume={11}, + number={118}, + pages={9344}, + year={2026}, + doi={10.21105/joss.09344}, +} + +@article{adeoye2026, + title={Self-concordant smoothing in proximal quasi-{Newton} algorithms for large-scale convex composite optimization}, + author={Adeoye, A. D. and Bemporad, Alberto}, + journal={Optimization Methods and Software}, + volume={41}, + number={3}, + pages={658--698}, + year={2026}, + doi={10.1080/10556788.2026.2647275}, +} + +@article{demarchi2023, + author={De~Marchi, Alberto and Jia, Xiaoxi and Kanzow, Christian and Mehlitz, Patrick}, + title={Constrained composite optimization and augmented {L}agrangian methods}, + journal={Mathematical Programming}, + year={2023}, + volume={201}, + number={1}, + pages={863--896}, + doi={10.1007/s10107-022-01922-4}, +} + +@article{pas2023, + title={Gauss--{Newton} meets {PANOC}: A fast and globally convergent algorithm for nonlinear optimal control}, + author={Pas, Pieter and Themelis, Andreas and Patrinos, Panagiotis}, + journal={IFAC-PapersOnLine}, + volume={56}, + number={2}, + pages={4852--4857}, + year={2023}, + note={22nd IFAC World Congress}, + doi={10.1016/j.ifacol.2023.10.1254}, +} + +@article{liang2025, + title={Primal-dual proximal bundle and conditional gradient methods for convex problems}, + author={Liang, Jiaming}, + journal={Mathematical Programming}, + volume={}, + number={}, + pages={}, + year={2025}, + doi={10.1007/s10107-025-02307-z}, +} + +@inproceedings{sathya2018, + title={Embedded Nonlinear Model Predictive Control for Obstacle Avoidance Using {PANOC}}, + author={Sathya, Ajay and Sopasakis, Pantelis and Van Parys, Ruben and Themelis, Andreas and Pipeleers, Goele and Patrinos, Panagiotis}, + booktitle={2018 European Control Conference (ECC)}, + year={2018}, + pages={1523--1528}, + doi={10.23919/ECC.2018.8550253}, +} + +@inproceedings{katriniok2019, + title={Nonlinear Model Predictive Control for Distributed Motion Planning in Road Intersections Using {PANOC}}, + author={Katriniok, Alexander and Sopasakis, Pantelis and Schuurmans, Mathijs and Patrinos, Panagiotis}, + booktitle={2019 IEEE 58th Conference on Decision and Control (CDC)}, + year={2019}, + pages={5272--5278}, + doi={10.1109/CDC40024.2019.9029703}, +} + +@article{besancon2022, + title={FrankWolfe.jl: A High-Performance and Flexible Toolbox for {Frank}--{Wolfe} Algorithms and Conditional Gradients}, + author={Besan\c{c}on, Mathieu and Carderera, Alejandro and Pokutta, Sebastian}, + journal={INFORMS Journal on Computing}, + volume={34}, + number={5}, + pages={2611--2620}, + year={2022}, + doi={10.1287/ijoc.2022.1191}, +} + +@article{waldmann2026, + title={A Proximal Multi-Objective Optimization Method for Incorporation of Polygenic Breeding Values in Genomic Prediction}, + author={Waldmann, Patrik and Fan, Yuhua}, + journal={IEEE Transactions on Computational Biology and Bioinformatics}, + year={2026}, + volume={23}, + number={1}, + pages={353--361}, + doi={10.1109/TCBBIO.2025.3642037}, +} + +@article{antonello2019, + title={Joint Acoustic Localization and Dereverberation Through Plane Wave Decomposition and Sparse Regularization}, + author={Antonello, Niccolò and De Sena, Enzo and Moonen, Marc and Naylor, Patrick A. and {van Waterschoot}, Toon}, + journal={IEEE/ACM Transactions on Audio, Speech, and Language Processing}, + year={2019}, + volume={27}, + number={12}, + pages={1893--1905}, + doi={10.1109/TASLP.2019.2933047}, +} + +@article{pyproximal2024, + title={PyProximal - scalable convex optimization in {Python}}, + author={Ravasi, Matteo and Örnhag, Marcus Valtonen and Luiken, Nick and Leblanc, Olivier and Uruñuela, Eneko}, + journal={Journal of Open Source Software}, + year={2024}, + volume={9}, + number={95}, + pages={6326}, + doi={10.21105/joss.06326}, +} + +@article{scico2022, + title={Scientific Computational Imaging Code (SCICO)}, + author={Balke, Thilo and Davis, Fernando and Garcia-Cardona, Cristina and Majee, Soumendu and McCann, Michael and Pfister, Luke and Wohlberg, Brendt}, + journal={Journal of Open Source Software}, + year={2022}, + volume={7}, + number={78}, + pages={4722}, + doi={10.21105/joss.04722}, +} + +@article{bergmann2022, + title={Manopt.jl: Optimization on Manifolds in {Julia}}, + author={Bergmann, Ronny}, + journal={Journal of Open Source Software}, + year={2022}, + volume={7}, + number={70}, + pages={3866}, + doi={10.21105/joss.03866}, +} + +@inproceedings{patrinos2013, + title={Proximal {Newton} methods for convex composite optimization}, + author={Patrinos, Panagiotis and Bemporad, Alberto}, + booktitle={52nd IEEE Conference on Decision and Control}, + pages={2358--2363}, + year={2013}, + doi={10.1109/CDC.2013.6760233}, +} + +@inproceedings{patrinos2014, + title={Douglas–{Rachford} splitting: Complexity estimates and accelerated variants}, + author={Patrinos, Panagiotis and Stella, Lorenzo and Bemporad, Alberto}, + booktitle={53rd IEEE Conference on Decision and Control}, + pages={4234--4239}, + year={2014}, + doi={10.1109/CDC.2014.7040049}, +} + +@inproceedings{pas2022, + title={Alpaqa: A matrix-free solver for nonlinear {MPC} and large-scale nonconvex optimization}, + author={Pas, Pieter and Schuurmans, Mathijs and Patrinos, Panagiotis}, + booktitle={2022 European Control Conference (ECC)}, + pages={417--422}, + year={2022}, + doi={10.23919/ECC55457.2022.9838172}, +} + +@article{sopasakis2020, + title={{OpEn}: Code generation for embedded nonconvex optimization}, + author={Sopasakis, Pantelis and Fresk, Emil and Patrinos, Panagiotis}, + journal={IFAC-PapersOnLine}, + volume={53}, + number={2}, + pages={6548--6554}, + year={2020}, + doi={10.1016/j.ifacol.2020.12.071}, +} diff --git a/paper/paper.md b/paper/paper.md new file mode 100644 index 0000000..ed3fce6 --- /dev/null +++ b/paper/paper.md @@ -0,0 +1,182 @@ +--- +title: 'ProximalAlgorithms.jl for nonsmooth optimization in Julia' +tags: + - Julia + - nonsmooth optimization + - nonconvex optimization + - convex optimization + - proximal algorithms + - splitting methods +authors: + - name: Lorenzo Stella + orcid: 0000-0002-8304-7327 + affiliation: 1 + - name: Niccolò Antonello + orcid: 0000-0002-0803-5385 + affiliation: 2 + - name: Alberto De Marchi + orcid: 0000-0002-3545-6898 + affiliation: 3 + corresponding: true + - name: Puya Latafat + orcid: 0000-0002-7969-8565 + affiliation: 4 + - name: Andreas Themelis + orcid: 0000-0002-6044-0169 + affiliation: 5 + - name: Panagiotis Patrinos + orcid: 0000-0003-4824-7697 + affiliation: 6 +affiliations: + - index: 1 + name: Amazon Web Services, Berlin, Germany (this work does not relate to the author's position at Amazon.) + - index: 2 + name: EssilorLuxottica, Milan, Italy (work done prior to joining EssilorLuxottica) + - index: 3 + name: University of the Bundeswehr Munich, Germany + - index: 4 + name: IMT School for Advanced Studies Lucca, Italy + - index: 5 + name: Kyushu University, Japan + - index: 6 + name: KU Leuven, Belgium +date: xx xxxxxxx 202x +bibliography: paper.bib +header-includes: | + \usepackage{booktabs} +--- + +# Summary + +[ProximalAlgorithms.jl](https://github.com/JuliaFirstOrder/ProximalAlgorithms.jl) is a Julia package that implements a broad collection of first-order splitting methods, commonly known as *proximal algorithms*, for solving structured nonsmooth optimization problems of the form +\begin{equation*} + \underset{x \in \mathbb{R}^n}{\text{minimize}} \quad \sum_{i=1}^{N} f_i(x), +\end{equation*} +where the number of terms $N$ depends on the algorithm, and each term $f_i$ may be smooth or nonsmooth, convex or, for some algorithms, nonconvex. +Since each $f_i$ may be extended-real valued, constraints can be encoded by means of indicator functions: given a feasible set $S$, $f_i$ can be defined as $f_i(x)=0$ if $x\in S$ and $f_i(x)=\infty$ otherwise. +Terms can also be composed with a linear mapping, as in $h(Lx)$, which arises frequently in imaging, signal processing, and control applications. +Proximal splitting methods exploit the problem structure and rely only on first-order information about each term: its gradient $\nabla f_i$, when differentiable, or its proximal mapping +\begin{equation*} + \operatorname{prox}_{\gamma f_i}(v) \in \underset{x \in \mathbb{R}^n}{\arg\min} \left\{ f_i(x) + \frac{1}{2\gamma}\|x - v\|^2 \right\} +\end{equation*} +for some stepsize $\gamma>0$. +This makes splitting methods well suited to large-scale problems in which nonsmooth terms (indicator functions of simple sets, sparsity-inducing penalties, norms, etc.) admit an inexpensive proximal mapping, even when the proximal mapping of their sum is difficult to compute. + +`ProximalAlgorithms.jl` organizes its solvers according to the structure of the problem they address: + +- **Two-term splitting** ($f + g$): (Fast) proximal gradient / forward-backward splitting [@lions1979; @beck2009], Douglas–Rachford splitting [@eckstein1992], and the linesearch-based methods ZeroFPR [@themelis2018], PANOC [@stella2017], PANOC+ [@demarchi2022], and Douglas–Rachford line search (DRLS) [@themelis2022]. +- **Three-term splitting** ($f + g + h$): the Davis–Yin splitting scheme [@davis2017]. +- **Primal-dual splitting** ($f + g + h \circ L$): Chambolle–Pock [@chambolle2011], Vũ–Condat [@vu2013; @condat2013], and the asymmetric forward-backward-adjoint (AFBA) method [@latafat2017]. + +Several linesearch methods build on the forward-backward and Douglas–Rachford envelopes [@patrinos2013; @patrinos2014]. +The package also includes the linesearch-free adaptive methods AdaPGM and AdaPDM [@latafat2025]. + +# Statement of need + +Nonsmooth structured optimization problems arise pervasively in statistics, machine learning, signal and image processing, and optimal control, wherever a differentiable loss or data-fidelity term is combined with nonsmooth regularizers or constraints. +Because methods differ in the term structure they support, in their convexity and smoothness assumptions, and in their empirical behavior, no single algorithm performs well across all instances. +Practitioners therefore need a large collection of solvers that can be swapped behind a common interface, so that algorithms can be compared and the best suited one selected for the problem at hand. +Researchers developing new methods, in turn, need a modular and extensible framework in which a novel solver can be implemented with little code, reuse shared components (gradient computation, proximal mappings, linesearch and stepsize strategies, termination handling), and be benchmarked fairly against existing methods. + +`ProximalAlgorithms.jl` addresses both needs: it provides a large toolbox of proximal and splitting solvers, and a modular, extensible framework for developing new ones. + +# Software design + +All solvers in `ProximalAlgorithms.jl` are implemented as Julia iterators, so that users can either call a solver directly to obtain a solution, or loop over the underlying iterator to gain fine-grained control over logging, stopping criteria, or warm-starting. +Gradients are obtained, by default, through automatic differentiation via [DifferentiationInterface.jl](https://github.com/gdalle/DifferentiationInterface.jl), which gives access to essentially every Julia automatic-differentiation backend, while proximal mappings follow the lightweight [ProximalCore.jl](https://github.com/JuliaFirstOrder/ProximalCore.jl) interface implemented by [ProximalOperators.jl](https://github.com/JuliaFirstOrder/ProximalOperators.jl) and other packages. +Users can just as easily supply hand-written gradients or proximal mappings for custom objective terms. + +`ProximalAlgorithms.jl` deliberately does not provide a modeling language that automatically decomposes an arbitrary objective into terms and picks a matching algorithm, in the way that, e.g., disciplined convex programming frameworks do for problems expressible in their grammar. +Instead, the user formulates the problem directly in terms of the objective terms ($f$, $g$, $h$, and, where relevant, a linear operator $L$) that a chosen algorithm expects, and the package supplies the iteration logic, termination handling, and (where applicable) adaptive stepsize with or without linesearch machinery. +This design keeps the package lightweight and composable: new algorithms can be added by implementing a small, self-contained iterator interface, and existing algorithms can be reused as subproblem solvers inside other packages. +[StructuredOptimization.jl](https://github.com/JuliaFirstOrder/StructuredOptimization.jl) provides a high-level modeling language and a user-friendly interface to invoke `ProximalAlgorithms.jl`. + +# State of the field + +`ProximalAlgorithms.jl` has been under development since October 2017, positioning it as one of the earlier general-purpose toolboxes offering a broad, unified interface to first-order splitting methods. +`ProximalAlgorithms.jl` gives users of the Julia optimization ecosystem direct access to a wide range of well-established, first-order splitting algorithms through a single, consistent, allocation-conscious interface, backed by `ProximalOperators.jl`s extensive library of proximable functions. + +This complements, rather than duplicates, other optimization software in the Julia ecosystem. +[RegularizedOptimization.jl](https://github.com/JuliaSmoothOptimizers/RegularizedOptimization.jl) [@gollier2026] implements model-based trust-region and quadratic-regularization methods for a similar class of nonsmooth problems. +Such methods typically require fewer evaluations of the smooth term and its gradient than the linesearch-based methods in `ProximalAlgorithms.jl`, at the cost of more proximal-operator evaluations. +[FrankWolfe.jl](https://github.com/ZIB-IOL/FrankWolfe.jl) is a toolbox for convex optimization using conditional gradient algorithms [@besancon2022], which rely on linear minimization oracles instead of proximal mappings. +[Manopt.jl](https://manoptjl.org/) [@bergmann2022] provides algorithms for optimization problems given on a Riemannian manifold and includes some splitting-based methods. + +There are also related Python packages that implement proximal methods, often restricted to convex problems or designed for specific applications, such as +[SCICO](https://github.com/lanl/scico) [@scico2022] and +[PyProximal](https://github.com/PyLops/pyproximal) [@pyproximal2024]. +Related software includes [alpaqa](https://github.com/kul-optec/alpaqa) [@pas2022] in C++, which combines an augmented Lagrangian framework with `PANOC` for constrained nonconvex optimization, and [OpEn](https://alphaville.github.io/optimization-engine/) [@sopasakis2020] in Rust, which generates solvers for embedded nonconvex optimization. + +# Research impact + +Since its initial release, `ProximalAlgorithms.jl` has grown both its user base and contributor community, while supporting research in control and signal processing as well as for advancing optimization methods. +@antonello2019 address joint acoustic source localization and dereverberation via sparse regularization. +In control, the `PANOC` solver has been used for embedded nonlinear model predictive control [@sathya2018], distributed motion planning at road intersections [@katriniok2019], and Gauss-Newton-accelerated nonlinear optimal control [@pas2023]. +Within optimization, @demarchi2023 builds on the `PANOCplus` solver to implement augmented Lagrangian methods for constrained nonsmooth problems. +The package has also seen adoption beyond its original developer community: +@liang2025 invokes `FISTA` as a subroutine for their numerical scheme, +@waldmann2026 use it to implement their proximal multi-objective method in genomic prediction, +@adeoye2026 benchmark their smoothing approach against several proximal-gradient algorithms in `ProximalAlgorithms.jl`. + +# Example + +The following condensed example minimizes the two-dimensional Rosenbrock function plus an $\ell_1$-norm regularizer, using the classical forward-backward splitting method: + +```julia +using ProximalOperators, ProximalAlgorithms +struct Rosenbrock{T} + a::T +end +(f::Rosenbrock)(x) = (f.a-x[1])^2 + 100*(x[2]-x[1]^2)^2 +function ProximalAlgorithms.value_and_gradient(f::Rosenbrock, x) + return f(x), [2*(x[1]-f.a) + 400*(x[1]^2-x[2])*x[1], 200*(x[2]-x[1]^2)] +end +f = Rosenbrock(1.0) +g = NormL1(2.0) +solver = ProximalAlgorithms.ForwardBackward(tol = 1e-6, verbose = true) +x, iters = solver(x0 = ones(2), f = f, g = g) +``` + +Here `f` wraps the smooth Rosenbrock term and `g` is the scaled $\ell_1$-norm, whose proximal mapping is the coordinate-wise soft-thresholding operator. +The `ForwardBackward` algorithm is instantiated with a termination tolerance and verbosity option, and then called with the initial point and the two objective terms to produce the solution and the iteration count. +Instead of providing the gradient of $f$, one can wrap the smooth cost for automatic differentiation, as with `f_auto`: + +```julia +using Zygote +using DifferentiationInterface: AutoZygote +f_auto = ProximalAlgorithms.AutoDifferentiable(x -> f(x), AutoZygote()) +x, iters = solver(x0 = ones(2), f = f_auto, g = g) +``` + +# Numerical illustration + +We compare proximal-gradient solvers `ForwardBackward`, `FastForwardBackward`, `ZeroFPR`, `PANOC`, and `PANOCplus` on a regularized line spectra estimation problem, which seeks to recover the frequencies and amplitudes of a signal that consists of a mixture of sinusoids. +The least-squares cost function $f$ is complemented with different regularizers $g$ from `ProximalOperators.jl`, either convex (`NormLinf`) or nonconvex (`IndBallL0`). +Experiments were performed with Ubuntu 64-bit on an Intel Core i7 (8-core) machine, using Julia 1.13.3. + +The tables report, for each regularizer, the success rate of each solver and the median values of runtime, the number of gradient and proximal evaluations, and the final objective value. +For `ZeroFPR`, `PANOC`, and `PANOCplus`, we use the default limited-memory BFGS Hessian approximation (memory 5). + +\begin{center} + `NormLinf` + + \input{examples/table_NormLinf.tex} + + `IndBallL0` + + \input{examples/table_IndBallL0.tex} +\end{center} + +Although the final objective values differ when the problem is nonconvex, all solvers successfully returned an approximate first-order stationary point, within the specified tolerance of $10^{-6}$, for all starting points. + +# Acknowledgements + +We acknowledge contributions from Ellis Brown, Guillaume Dalle, William Kong, Paulo J. S. Silva, and Tamás Hakkel. + +# AI usage disclosure + +Generative AI tools were not used to develop the software. +Claude Sonnet 5.5 was adopted to improve readability and proofread this text. +All AI-generated suggestions were critically reviewed, modified where necessary, and fully validated by the authors, who retain complete responsibility. + +# References