From e51cc976d246b77cfab7f5708d6155035da7a12e Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Thu, 25 Jul 2024 11:51:00 +0200 Subject: [PATCH 1/7] implement variable squashing this should hopefully save some optimizer effort in cases where large parts of constraint systems get eliminated --- src/constraint_tree.jl | 63 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/src/constraint_tree.jl b/src/constraint_tree.jl index 2fb4a03..d9061f2 100644 --- a/src/constraint_tree.jl +++ b/src/constraint_tree.jl @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import DataStructures: SortedDict +import DataStructures: SortedDict, SortedSet """ $(TYPEDEF) @@ -98,7 +98,7 @@ $(TYPEDSIGNATURES) Find the expected count of variables in a [`ConstraintTree`](@ref). """ -var_count(x::ConstraintTree) = isempty(elems(x)) ? 0 : maximum(var_count.(values(elems(x)))) +var_count(x::ConstraintTree) = isempty(elems(x)) ? 0 : maximum(var_count.(values(x))) """ $(TYPEDSIGNATURES) @@ -159,6 +159,65 @@ incr_var_idxs(x::QuadraticValue, incr::Int) = QuadraticValue( weights = x.weights, ) +""" +$(TYPEDSIGNATURES) + +Push all variable indexes found in `x` to the `out` container. + +(The container needs to support the standard `push!`.) +""" +collect_variables!(x::Constraint, out) = collect_variables!(x.value) +collect_variables!(x::LinearValue, out) = + for idx in x.idxs + push!(out, idx) + end +collect_variables!(x::QuadraticValue, out) = + for (idx, idy) in x.idxs + push!(out, idx, idy) + end +collect_variables!(x::ConstraintTree, out) = collect_variables!.(values(x), Ref(out)) + +""" +$(TYPEDSIGNATURES) + +Prune the unused variable indexes from an object `x` (such as a +[`ConstraintTree`](@ref)). + +This first runs [`collect_variables!`](@ref) to determine the actual used +variables, then calls [`renumber_variables`](@ref) to create a renumbered +object. +""" +function prune_variables(x) + vars = SortedSet{Int}() + collect_variables!(x, vars) + push!(vars, 0) + vv = collect(vars) + @assert vv[1] == 0 "variable indexes are broken" + return renumber_variables(x, SortedDict(vv .=> 0:length(vv))) +end + +""" +$(TYPEDSIGNATURES) + +Renumber all variables in an object (such as [`ConstraintTree`](@ref)). The new +variable indexes are taken from the `mapping` parameter at the index of the old +variable's index. + +This does not run any consistency checks on the result; the `mapping` must +therefore be monotonically increasing, and the zero index must map to itself, +otherwise invalid [`Value`](@ref)s will be produced. +""" +renumber_variables(x::ConstraintTree, mapping) = + ConstraintTree(k => renumber_Variables(v, mapping) for (k, v) in x) +renumber_variables(x::Constraint, mapping) = + Constraint(renumber_variables(x.value, mapping), x.bound) +renumber_variables(x::LinearValue, mapping) = + LinearValue(idxs = [mapping[idx] for idx in x.idxs], weights = x.weights) +renumber_variables(x::QuadraticValue, mapping) = QuadraticValue( + idxs = [(mapping[idx], mapping[idy]) for (idx, idy) in x.idxs], + weights = x.weights, +) + # # Algebraic construction # From fe9877d3fafeedad247e2f37218f52abf2b9680b Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Thu, 25 Jul 2024 11:52:16 +0200 Subject: [PATCH 2/7] this will require a bump --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 058c84b..eee3c7d 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ConstraintTrees" uuid = "5515826b-29c3-47a5-8849-8513ac836620" authors = ["The developers of ConstraintTrees.jl"] -version = "1.2.0" +version = "1.3.0" [deps] ConstructionBase = "187b0558-2788-49d3-abe0-74a17ed4e7c9" From a731dd7f3f2bdc90bd69d713ccecbebea2382b30 Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Thu, 25 Jul 2024 14:37:26 +0200 Subject: [PATCH 3/7] implement the filtering functions Closes #34 --- src/tree.jl | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/tree.jl b/src/tree.jl index acd4b67..3c2cfc5 100644 --- a/src/tree.jl +++ b/src/tree.jl @@ -156,6 +156,58 @@ end """ $(TYPEDSIGNATURES) +Filter all branches and leaves in a tree, leaving only the ones where `f` +returns `true`. + +Note that the branches are passed to `f` as well. Use [`filter_lea2es`](@ref) +to only work with the leaf values. +""" +function filter(f, x::Tree{T}) where {T} + go(x::Tree) = Tree{T}(k => go(v) for (k, v) in x if f(v)) + go(x) = f(x) + + go(x) +end + +""" +$(TYPEDSIGNATURES) + +Like [`filter`](@ref) but the filtering predicate function also receives the +"path" in the tree. +""" +function ifilter(f, x::Tree{T}) where {T} + go(ix, x::Tree{T}) = Tree{T}(k => go(tuple(ix..., k), v) for (k, v) in x if f(v)) + go(ix, x) = f(ix, x) + + go((), x) +end + +""" +$(TYPEDSIGNATURES) + +Like [`filter`](@ref) but the filtering predicate function `f` only receives +the leaf values (i.e., no intermediate sub-trees). + +In turn, the result will retain the whole subtree structure (even if empty). +""" +filter_leaves(f, x::Tree{T}) where {T} = + let flt(x::Tree{T}) = true, flt(x) = f(x) + filter(flt, x) + end + +""" +$(TYPEDSIGNATURES) + +Combination of [`ifilter`](@ref) and [`filter_leaves`](@ref). +""" +ifilter_leaves(f, x::Tree{T}) where {T} = + let flt(_, x::Tree{T}) = true, flt(i, x) = f(i, x) + ifilter(flt, x) + end + +""" +$(TYPEDSIGNATURES) + Like [`map`](@ref), but discards the results, thus relying only on the side effects of `f`. From cc7c187c9d25973cdbf627794a2aa461bfdd64aa Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Thu, 25 Jul 2024 14:40:27 +0200 Subject: [PATCH 4/7] typo --- src/tree.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tree.jl b/src/tree.jl index 3c2cfc5..065154c 100644 --- a/src/tree.jl +++ b/src/tree.jl @@ -159,7 +159,7 @@ $(TYPEDSIGNATURES) Filter all branches and leaves in a tree, leaving only the ones where `f` returns `true`. -Note that the branches are passed to `f` as well. Use [`filter_lea2es`](@ref) +Note that the branches are passed to `f` as well. Use [`filter_leaves`](@ref) to only work with the leaf values. """ function filter(f, x::Tree{T}) where {T} From 6d694e057b96e789c048b1ac5afb5e9f81c4af5e Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Thu, 25 Jul 2024 15:20:25 +0200 Subject: [PATCH 5/7] Fix stuff, add tests&docs. The lack of static typechecker stays disturbing. --- docs/src/4-functional-tree-processing.jl | 80 ++++++++++++++++++++++-- src/constraint_tree.jl | 6 +- src/tree.jl | 17 ++--- 3 files changed, 89 insertions(+), 14 deletions(-) diff --git a/docs/src/4-functional-tree-processing.jl b/docs/src/4-functional-tree-processing.jl index 030244c..ca41928 100644 --- a/docs/src/4-functional-tree-processing.jl +++ b/docs/src/4-functional-tree-processing.jl @@ -42,9 +42,10 @@ # - [`variables_for`](@ref ConstraintTrees.variables_for) allocates a variable # for each constraint in the tree and allows the user to specify bounds # -# Additionally, all these have their "indexed" variant which allows you to know -# the path where the tree elements are being merged. The path is passed to the -# handling function as a tuple of symbols. The variants are prefixed with `i`: +# Additionally, all these have their "indexed" variant which allows the user to +# know the path where the tree elements are being merged. The path is passed to +# the handling function as a tuple of symbols. The variants are prefixed with +# `i`: # # - [`imap`](@ref ConstraintTrees.imap) # - [`imapreduce`](@ref ConstraintTrees.ireduce) (here the path refers to the @@ -202,7 +203,8 @@ tz.y @test isapprox(tz.y.x, 0.99) #src @test isapprox(tz.y.y, 0.78) #src -# We also have the indexed variants; for example this allows us to only merge the `x` elements in points: +# We also have the indexed variants; for example this allows us to only merge +# the `x` elements in points: tx = C.imerge(t1, t2, Float64) do path, x, y last(path) == :x || return missing @@ -313,3 +315,73 @@ end; # To prevent uncertainty, both functions always traverse the keys in sorted # order. + +# ## Removing constraints with `filter` +# +# In many cases it is beneficial to simplify the constraint system by +# systematically removing constraints. [`filter`](@ref ConstraintTrees.filter) +# and [`ifilter`](@ref ConstraintTrees.ifilter) run a function on all subtrees +# and leaves (usually the leaves are [`Constraint`](@ref +# ConstraintTrees.Constraint)s), and only retain these where the function +# returns `true`. +# +# For example, this removes all constraints named `y`: + +filtered = C.ifilter(x) do ix, c + return c isa C.ConstraintTree || last(ix) != :y +end + +filtered.z + +@test !haskey(filtered.x, :y) #src + +# Functions [`filter_leaves`](@ref ConstraintTrees.filter_leaves) and +# [`ifilter_leaves`](@ref ConstraintTrees.ifilter_leaves) act similarly but +# automatically assume that the directory structure is going to stay intact, +# freeing the user from having to handle the subdirectories. +# +# The above example thus simplifies to: + +filtered = C.ifilter_leaves(x) do ix, c + last(ix) != :y +end + +filtered.z + +@test !haskey(filtered.x, :y) #src + +# We can also remove whole variable ranges: + +filtered = C.filter_leaves(x) do c + all(>=(4), c.value.idxs) +end + +# Notably, these operations leave the tree with a slightly sub-optimal state, +# as there are indexes allocated for variables that are no longer used! + +C.var_count(filtered) + +# To fix the issue, it is possible to "squash" the variable indexes using +# [`prune_variables`](@ref ConstraintTrees.prune_variables): + +pruned = C.prune_variables(filtered) + +C.var_count(pruned) + +@test C.var_count(pruned) == 3 #src + +# Note that given the renumbering, constraint trees are no longer compatible +# after pruning, and should not be combined with `*`. As an anti-example, one +# might be interested in pruning the variable values before joining them in to +# larger constraint tree, e.g. to simplify larger quadratic values: + +pruned_qv = C.prune_variables(x.y.x.value * x.z.y.value) + +# Despite the variable count decreased, the value now corresponds to a +# completely different value in the original tree! Compare: + +(pruned_qv, x.x.x.value * x.x.y.value) + +@test C.var_count(pruned_qv) == 2 #src +@test pruned_qv.idxs == (x.x.x.value * x.x.y.value).idxs #src +@test pruned_qv.weights == (x.x.x.value * x.x.y.value).weights #src diff --git a/src/constraint_tree.jl b/src/constraint_tree.jl index d9061f2..c7853e8 100644 --- a/src/constraint_tree.jl +++ b/src/constraint_tree.jl @@ -166,7 +166,7 @@ Push all variable indexes found in `x` to the `out` container. (The container needs to support the standard `push!`.) """ -collect_variables!(x::Constraint, out) = collect_variables!(x.value) +collect_variables!(x::Constraint, out) = collect_variables!(x.value, out) collect_variables!(x::LinearValue, out) = for idx in x.idxs push!(out, idx) @@ -193,7 +193,7 @@ function prune_variables(x) push!(vars, 0) vv = collect(vars) @assert vv[1] == 0 "variable indexes are broken" - return renumber_variables(x, SortedDict(vv .=> 0:length(vv))) + return renumber_variables(x, SortedDict(vv .=> 0:length(vv)-1)) end """ @@ -208,7 +208,7 @@ therefore be monotonically increasing, and the zero index must map to itself, otherwise invalid [`Value`](@ref)s will be produced. """ renumber_variables(x::ConstraintTree, mapping) = - ConstraintTree(k => renumber_Variables(v, mapping) for (k, v) in x) + ConstraintTree(k => renumber_variables(v, mapping) for (k, v) in x) renumber_variables(x::Constraint, mapping) = Constraint(renumber_variables(x.value, mapping), x.bound) renumber_variables(x::LinearValue, mapping) = diff --git a/src/tree.jl b/src/tree.jl index 065154c..ae9597b 100644 --- a/src/tree.jl +++ b/src/tree.jl @@ -164,7 +164,7 @@ to only work with the leaf values. """ function filter(f, x::Tree{T}) where {T} go(x::Tree) = Tree{T}(k => go(v) for (k, v) in x if f(v)) - go(x) = f(x) + go(x) = x go(x) end @@ -176,8 +176,9 @@ Like [`filter`](@ref) but the filtering predicate function also receives the "path" in the tree. """ function ifilter(f, x::Tree{T}) where {T} - go(ix, x::Tree{T}) = Tree{T}(k => go(tuple(ix..., k), v) for (k, v) in x if f(v)) - go(ix, x) = f(ix, x) + go(ix, x::Tree{T}) = + Tree{T}(k => go(tuple(ix..., k), v) for (k, v) in x if f(tuple(ix..., k), v)) + go(ix, x) = x go((), x) end @@ -190,10 +191,12 @@ the leaf values (i.e., no intermediate sub-trees). In turn, the result will retain the whole subtree structure (even if empty). """ -filter_leaves(f, x::Tree{T}) where {T} = - let flt(x::Tree{T}) = true, flt(x) = f(x) - filter(flt, x) - end +function filter_leaves(f, x::Tree{T}) where {T} + flt(x::Tree{T}) = true + flt(x) = f(x) + + filter(flt, x) +end """ $(TYPEDSIGNATURES) From 54f4b3d140a841dd5919436ade16adee79ca1228 Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Thu, 25 Jul 2024 15:52:59 +0200 Subject: [PATCH 6/7] now we're at this, implement zero dropping --- docs/src/4-functional-tree-processing.jl | 43 +++++++++++++++++++----- src/constraint_tree.jl | 22 +++++++++--- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/docs/src/4-functional-tree-processing.jl b/docs/src/4-functional-tree-processing.jl index ca41928..2c2de4f 100644 --- a/docs/src/4-functional-tree-processing.jl +++ b/docs/src/4-functional-tree-processing.jl @@ -356,8 +356,11 @@ filtered = C.filter_leaves(x) do c all(>=(4), c.value.idxs) end -# Notably, these operations leave the tree with a slightly sub-optimal state, -# as there are indexes allocated for variables that are no longer used! +# ### Pruning unused variable references +# +# Filtering operations may leave the constraint tree in a slightly sub-optimal +# state, where there are indexes allocated for variables that are no longer +# used! C.var_count(filtered) @@ -370,18 +373,42 @@ C.var_count(pruned) @test C.var_count(pruned) == 3 #src -# Note that given the renumbering, constraint trees are no longer compatible -# after pruning, and should not be combined with `*`. As an anti-example, one -# might be interested in pruning the variable values before joining them in to -# larger constraint tree, e.g. to simplify larger quadratic values: +# Note that after the pruning and renumbering, the involved constraint trees +# are no longer compatible, and should not be combined with `*`. As an +# anti-example, one might be interested in pruning the variable values before +# joining them in to larger constraint tree, e.g. to simplify larger quadratic +# values: pruned_qv = C.prune_variables(x.y.x.value * x.z.y.value) -# Despite the variable count decreased, the value now corresponds to a -# completely different value in the original tree! Compare: +# This value now corresponds to a completely different value in the original +# tree! Compare: (pruned_qv, x.x.x.value * x.x.y.value) @test C.var_count(pruned_qv) == 2 #src @test pruned_qv.idxs == (x.x.x.value * x.x.y.value).idxs #src @test pruned_qv.weights == (x.x.x.value * x.x.y.value).weights #src + +# As another common source of redundant variable references, some variables may +# be used with zero weights. This situation is not detected by +# [`prune_variables`](@ref ConstraintTrees.prune_variables) by default, but you +# can remove the "zeroed out" variable references by using +# [`drop_zeros`](@ref), which allows the pruning to work properly. +# +# For example, the value constructed in the tree below does not really refer to +# `x.x.y` anymore, but pruning does not help to get rid of the now-redundant +# variable: + +x.x.y.value = x.x.y.value + x.x.x.value * x.x.x.value - x.x.y.value + +C.var_count(C.prune_variables(x)) + +@test C.var_count(C.prune_variables(x)) == 6 #src + +# After the zero-weight variable references are dropped, the pruning behaves as +# desired: + +C.var_count(C.prune_variables(C.drop_zeros(x))) + +@test C.var_count(C.prune_variables(C.drop_zeros(x))) == 5 #src diff --git a/src/constraint_tree.jl b/src/constraint_tree.jl index c7853e8..e93a0fc 100644 --- a/src/constraint_tree.jl +++ b/src/constraint_tree.jl @@ -98,7 +98,7 @@ $(TYPEDSIGNATURES) Find the expected count of variables in a [`ConstraintTree`](@ref). """ -var_count(x::ConstraintTree) = isempty(elems(x)) ? 0 : maximum(var_count.(values(x))) +var_count(x::ConstraintTree) = isempty(x) ? 0 : maximum(var_count.(values(x))) """ $(TYPEDSIGNATURES) @@ -130,7 +130,7 @@ Offset all variable indexes in a [`ConstraintTree`](@ref) by the given increment. """ incr_var_idxs(x::ConstraintTree, incr::Int) = - ConstraintTree(k => incr_var_idxs(v, incr) for (k, v) in elems(x)) + ConstraintTree(k => incr_var_idxs(v, incr) for (k, v) in x) """ $(TYPEDSIGNATURES) @@ -175,7 +175,8 @@ collect_variables!(x::QuadraticValue, out) = for (idx, idy) in x.idxs push!(out, idx, idy) end -collect_variables!(x::ConstraintTree, out) = collect_variables!.(values(x), Ref(out)) +collect_variables!(x::Tree{T}, out::C) where {T,C} = + collect_variables!.(values(x), Ref(out)) """ $(TYPEDSIGNATURES) @@ -207,7 +208,7 @@ This does not run any consistency checks on the result; the `mapping` must therefore be monotonically increasing, and the zero index must map to itself, otherwise invalid [`Value`](@ref)s will be produced. """ -renumber_variables(x::ConstraintTree, mapping) = +renumber_variables(x::Tree{T}, mapping) where {T} = ConstraintTree(k => renumber_variables(v, mapping) for (k, v) in x) renumber_variables(x::Constraint, mapping) = Constraint(renumber_variables(x.value, mapping), x.bound) @@ -218,6 +219,19 @@ renumber_variables(x::QuadraticValue, mapping) = QuadraticValue( weights = x.weights, ) +""" +$(TYPEDSIGNATURES) + +Remove variable references from all [`Value`](@ref)s in the given object +(usually a [`ConstraintTree`](@ref)) where the variable weight is exactly zero. +""" +drop_zeros(x::Tree{T}) where {T} = ConstraintTree(k => drop_zeros(v) for (k, v) in x) +drop_zeros(x::Constraint) = Constraint(drop_zeros(x.value), x.bound) +drop_zeros(x::LinearValue) = + LinearValue(idxs = x.idxs[x.weights.!=0], weights = x.weights[x.idxs.!=0]) +drop_zeros(x::QuadraticValue) = + QuadraticValue(idxs = x.idxs[x.weights.!=0], weights = x.weights[x.weights.!=0]) + # # Algebraic construction # From a9714cf5f9e99e1bcc37f83966ad39d915cfcb92 Mon Sep 17 00:00:00 2001 From: Mirek Kratochvil Date: Thu, 25 Jul 2024 16:02:49 +0200 Subject: [PATCH 7/7] fix doc ref --- docs/src/4-functional-tree-processing.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/src/4-functional-tree-processing.jl b/docs/src/4-functional-tree-processing.jl index 2c2de4f..705be72 100644 --- a/docs/src/4-functional-tree-processing.jl +++ b/docs/src/4-functional-tree-processing.jl @@ -394,7 +394,8 @@ pruned_qv = C.prune_variables(x.y.x.value * x.z.y.value) # be used with zero weights. This situation is not detected by # [`prune_variables`](@ref ConstraintTrees.prune_variables) by default, but you # can remove the "zeroed out" variable references by using -# [`drop_zeros`](@ref), which allows the pruning to work properly. +# [`drop_zeros`](@ref ConstraintTrees.drop_zeros), which allows the pruning to +# work properly. # # For example, the value constructed in the tree below does not really refer to # `x.x.y` anymore, but pruning does not help to get rid of the now-redundant