Improve test suite performance

test/runtests.jl

@@ -67,9 +67,6 @@ include("test-in-place-residual.jl")
 @everywhere function test_one_problem(prob::Symbol)
   pb = string(prob)
 
-  nvar = OptimizationProblems.eval(Symbol(:get_, prob, :_nvar))()
-  ncon = OptimizationProblems.eval(Symbol(:get_, prob, :_ncon))()
-
   function timed_info(label, f, args...; kwargs...)
     stats = @timed f(args...; kwargs...)
     msg =
@@ -82,29 +79,28 @@ include("test-in-place-residual.jl")
 
   nlp_ad = timed_info("Instantiating $(pb)", make_nlp, prob)
 
-  @testset "Sanity check (name, obj)" begin
+  @testset "$pb" begin
     @test nlp_ad.meta.name == pb
     @test !isnothing(obj(nlp_ad, nlp_ad.meta.x0))
-  end
 
-  if (typeof(nlp_ad) <: ADNLPModels.AbstractADNLPModel) &&
-     (pb in meta[(meta.contype .== :quadratic) .| (meta.contype .== :general), :name])
-    @testset "Test In-place Nonlinear Constraints for AD-$prob" begin
+    if (typeof(nlp_ad) <: ADNLPModels.AbstractADNLPModel) &&
+       (pb in meta[(meta.contype .== :quadratic) .| (meta.contype .== :general), :name])
       test_in_place_constraints(prob, nlp_ad)
     end
-  end
 
-  if typeof(nlp_ad) <: ADNLPModels.AbstractADNLPModel
-    @testset "Test multi-precision ADNLPProblems for $prob" begin
+    if typeof(nlp_ad) <: ADNLPModels.AbstractADNLPModel
       test_multi_precision(prob, nlp_ad)
     end
-  end
 
-  if prob in intersect(list_problems_PureJuMP, list_problems_ADNLPProblems)
-    @testset "Test problems compatibility for $prob" begin
+    if prob in intersect(list_problems_PureJuMP, list_problems_ADNLPProblems)
       nlp_jump = make_jump_nlp(prob; n = ndef)
       test_compatibility(prob, nlp_jump, nlp_ad, ndef)
     end
+
+    if prob in nls_name_list
+      nls_model = make_ad_nlp(prob; use_nls = true)
+      test_in_place_residual(prob, nlp_ad, nls_model)
+    end
   end
 end
 

test/test-defined-problems.jl

@@ -8,45 +8,23 @@
   return (pid = myid(), missing = missing)
 end
 
-probes = @sync begin
-  for pid in workers()
-    @async remotecall_fetch(probe_missing, pid, ADNLPProblems, list_problems_ADNLPProblems)
-  end
-end
-@info "ADNLPProblems missing per worker" probes
-
-probes = @sync begin
-  for pid in workers()
-    @async remotecall_fetch(probe_missing, pid, PureJuMP, list_problems_PureJuMP)
+if get(ENV, "JULIA_TEST_DEBUG", "") == "1"
+  probes = @sync begin
+    for pid in workers()
+      @async remotecall_fetch(probe_missing, pid, ADNLPProblems, list_problems_ADNLPProblems)
+    end
   end
-end
-@info "PureJuMP missing per worker" probes
-
-@testset "Adjusted dimension warnings" begin
-  var_probs = OptimizationProblems.meta[OptimizationProblems.meta.variable_nvar, :name]
+  @info "ADNLPProblems missing per worker" probes
 
-  for prob_name in var_probs
-    prob_sym = Symbol(prob_name)
-
-    get_nvar_func = getfield(OptimizationProblems, Symbol("get_", prob_name, "_nvar"))
-
-    for n in (50, 100)
-      n_adjusted = get_nvar_func(; n = n)
-      n_adjusted == n && continue  # Skip if no adjustment for this n
-
-      msg_re = Regex("number of variables adjusted from $(n) to $(n_adjusted)")
-
-      for mod in (ADNLPProblems, PureJuMP)
-        isdefined(mod, prob_sym) || continue
-        constructor = getfield(mod, prob_sym)
-        @test_logs (:warn, msg_re) constructor(; n = n)
-      end
-
-      break
+  probes = @sync begin
+    for pid in workers()
+      @async remotecall_fetch(probe_missing, pid, PureJuMP, list_problems_PureJuMP)
     end
   end
+  @info "PureJuMP missing per worker" probes
 end
 
+
 @test setdiff(union(names(ADNLPProblems), list_problems_not_ADNLPProblems), list_problems) ==
       [:ADNLPProblems]
 @test setdiff(union(names(PureJuMP), list_problems_not_PureJuMP), list_problems) == [:PureJuMP]

test/test-in-place-residual.jl

@@ -11,19 +11,16 @@ end
   nlp::AbstractNLPModel,
   nls::AbstractNLSModel,
 )
-  @testset "Test in-place residual $prob" begin
+  @testset "NLS $prob" begin
     x = nls.meta.x0
     Fx = similar(x, nls.nls_meta.nequ)
     pb = String(prob)
     if VERSION ≥ v"1.7" && !occursin("palmer", pb) && (pb != "watson") # palmer residual allocate
       @allocated residual!(nls, x, Fx)
       @test (@allocated residual!(nls, x, Fx)) == 0
     end
-    m = OptimizationProblems.eval(Meta.parse("get_$(prob)_nls_nequ"))()
+    m = getfield(OptimizationProblems, Symbol(:get_, prob, :_nls_nequ))()
     @test nls.nls_meta.nequ == m
-  end
-
-  @testset "Compare NLS with NLP $prob: x0 and obj are the same." begin
     x0 = nlp.meta.x0
     @test x0 == nls.meta.x0
     nlp_fx = obj(nlp, x0)
@@ -36,6 +33,5 @@ end
   end
 end
 
-nls_name_list =
+@everywhere const nls_name_list =
   intersect(Symbol.(meta[meta.objtype .== :least_squares, :name]), list_problems_ADNLPProblems)
-pmap(test_in_place_residual, nls_name_list)

test/test-scalable.jl

@@ -1,17 +1,43 @@
 @everywhere function test_scalable(item)
-  name = item.name
+  name = item.name  # Symbol
   @testset "Test scalable problems - problem: $(name)" begin
-    nlp = make_nlp(Symbol(name))
+    nlp = make_nlp(name)
     @test item.nvar == nlp.meta.nvar
-    nlp11 = make_nlp(Symbol(name); n = 13 * ndef)
-    n11 = OptimizationProblems.eval(Symbol(:get_, name, :_nvar))(n = 13 * ndef)
+    nlp11 = make_nlp(name; n = 13 * ndef)
+    n11 = getfield(OptimizationProblems, Symbol(:get_, name, :_nvar))(n = 13 * ndef)
     @test n11 == nlp11.meta.nvar
 
     # test that the problem is actually scalable
     @test n11 != item.nvar
+
+    # dimension-adjustment warning checks (folded in from test-defined-problems.jl)
+    for (n, is_adnlp) in item.warn_checks
+      mod = is_adnlp ? ADNLPProblems : PureJuMP
+      n_adjusted = getfield(OptimizationProblems, Symbol(:get_, name, :_nvar))(; n = n)
+      msg_re = Regex("number of variables adjusted from $(n) to $(n_adjusted)")
+      @testset "Adjusted dim warning $name ($(nameof(mod))) n=$n" begin
+        @test_logs (:warn, msg_re) getfield(mod, name)(; n = n)
+      end
+    end
   end
 end
 
 names_pb_vars = meta[meta.variable_nvar .== true, [:nvar, :name]]
-items = [(name = Symbol(row[:name]), nvar = row[:nvar]) for row in eachrow(names_pb_vars)]
+items = map(eachrow(names_pb_vars)) do row
+  prob_name = row[:name]
+  prob_sym = Symbol(prob_name)
+  get_nvar_func = getfield(OptimizationProblems, Symbol("get_", prob_name, "_nvar"))
+  warn_checks = Tuple{Int, Bool}[]
+  for n in (50, 100)
+    n_adjusted = get_nvar_func(; n = n)
+    n_adjusted == n && continue
+    for is_adnlp in (true, false)
+      mod = is_adnlp ? ADNLPProblems : PureJuMP
+      isdefined(mod, prob_sym) || continue
+      push!(warn_checks, (n, is_adnlp))
+    end
+    break
+  end
+  (name = prob_sym, nvar = row[:nvar], warn_checks = warn_checks)
+end
 pmap(test_scalable, items)

test/test-utils.jl

@@ -7,18 +7,18 @@
 @everywhere meta = OptimizationProblems.meta
 
 @everywhere function meta_sanity_check(prob::Symbol, nlp::AbstractNLPModel)
-  meta = OptimizationProblems.eval(Symbol(prob, :_meta))
-  getnvar = OptimizationProblems.eval(Symbol(:get_, prob, :_nvar))(n = test_nvar)
+  meta = getfield(OptimizationProblems, Symbol(prob, :_meta))
+  getnvar = getfield(OptimizationProblems, Symbol(:get_, prob, :_nvar))(n = test_nvar)
   @test getnvar == meta[:nvar] || meta[:variable_nvar]
-  getncon = OptimizationProblems.eval(Symbol(:get_, prob, :_ncon))(n = test_nvar)
+  getncon = getfield(OptimizationProblems, Symbol(:get_, prob, :_ncon))(n = test_nvar)
   @test getncon == meta[:ncon] || meta[:variable_ncon]
-  getnlin = OptimizationProblems.eval(Symbol(:get_, prob, :_nlin))(n = test_nvar)
+  getnlin = getfield(OptimizationProblems, Symbol(:get_, prob, :_nlin))(n = test_nvar)
   @test getnlin == nlp.meta.nlin || meta[:variable_ncon]
-  getnnln = OptimizationProblems.eval(Symbol(:get_, prob, :_nnln))(n = test_nvar)
+  getnnln = getfield(OptimizationProblems, Symbol(:get_, prob, :_nnln))(n = test_nvar)
   @test getnnln == nlp.meta.nnln || meta[:variable_ncon]
-  getnequ = OptimizationProblems.eval(Symbol(:get_, prob, :_nequ))(n = test_nvar)
+  getnequ = getfield(OptimizationProblems, Symbol(:get_, prob, :_nequ))(n = test_nvar)
   @test getnequ == length(get_jfix(nlp)) || meta[:variable_ncon]
-  getnineq = OptimizationProblems.eval(Symbol(:get_, prob, :_nineq))(n = test_nvar)
+  getnineq = getfield(OptimizationProblems, Symbol(:get_, prob, :_nineq))(n = test_nvar)
   @test getnineq == (get_ncon(nlp) - length(get_jfix(nlp))) || meta[:variable_ncon]
   @test meta[:best_known_lower_bound] <= meta[:best_known_upper_bound]
   @test meta[:minimize] == get_minimize(nlp)
@@ -29,7 +29,7 @@
 end
 
 @everywhere function test_in_place_constraints(prob::Symbol)
-  nlp = OptimizationProblems.ADNLPProblems.eval(prob)()
+  nlp = getfield(ADNLPProblems, prob)()
   return test_in_place_constraints(prob, nlp)
 end
 
@@ -42,20 +42,15 @@ end
     @allocated cons_nln!(nlp, x, cx)
     @test (@allocated cons_nln!(nlp, x, cx)) == 0
   end
-  m = OptimizationProblems.eval(Meta.parse("get_$(prob)_nnln"))()
+  m = getfield(OptimizationProblems, Symbol(:get_, prob, :_nnln))()
   @test ncon == m
 end
 
 @everywhere function test_compatibility(prob::Symbol, ndef::Integer = ndef)
-  prob_fn = eval(Meta.parse("PureJuMP.$(prob)"))
+  prob_fn = getfield(PureJuMP, prob)
   model = prob_fn(n = ndef)
   nlp_jump = MathOptNLPModel(model)
-
-  nvar = OptimizationProblems.eval(Symbol(:get_, prob, :_nvar))()
-  ncon = OptimizationProblems.eval(Symbol(:get_, prob, :_ncon))()
-
   nlp_ad = make_ad_nlp(prob)
-
   return test_compatibility(prob, nlp_jump, nlp_ad, ndef)
 end
 
@@ -112,9 +107,6 @@ end
 end
 
 @everywhere function test_multi_precision(prob::Symbol; list_types = [Float32, Float64])
-  nvar = OptimizationProblems.eval(Symbol(:get_, prob, :_nvar))()
-  ncon = OptimizationProblems.eval(Symbol(:get_, prob, :_ncon))()
-
   for T in list_types
     nlp = make_ad_nlp(prob; type = T)
     test_multi_precision(T, nlp)