Merge pull request #159 from JuliaRobotics/master

Release 0.4.1
JuliaRobotics · Oct 14, 2019 · 838d83b · 838d83b
2 parents 735e8ec + 3191d82
commit 838d83b
Show file tree

Hide file tree

Showing 33 changed files with 1,102 additions and 516 deletions.
diff --git a/Project.toml b/Project.toml
@@ -7,12 +7,12 @@ Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 JSON2 = "2535ab7d-5cd8-5a07-80ac-9b1792aadce3"
 LightGraphs = "093fc24a-ae57-5d10-9952-331d41423f4d"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MetaGraphs = "626554b9-1ddb-594c-aa3c-2596fe9399a5"
-Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
@@ -28,6 +28,8 @@ julia = "0.7, 1"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+GraphPlot = "a2cc645c-3eea-5389-862e-a155d0052231"
+Neo4j = "d2adbeaf-5838-5367-8a2f-e46d570981db"
 
 [targets]
-test = ["Test"]
+test = ["Test", "GraphPlot", "Neo4j"]
diff --git a/docs/src/func_ref.md b/docs/src/func_ref.md
@@ -58,8 +58,15 @@ getSubgraphAroundNode
 getSubgraph
 ```
 
-### Visualization
+### Summaries
+```@docs
+getSummary
+getSummaryGraph
+```
+
+### Visualization and Plotting
 ```@docs
 toDot
 toDotFile
+dfgplot
 ```
diff --git a/docs/src/variable_factor_serialization.md b/docs/src/variable_factor_serialization.md
@@ -0,0 +1,68 @@
+# Serialization of Variables and Factors
+
+If you are transferring variables and factors over a wire you need to serialize
+and deserialize variables and factors.
+
+## Packing and Unpacking
+
+Packing is done with the exposed functions `packVariable()::Dict{String, Any}` and
+`packFactor()::Dict{String, Any}`. You can then serialize this into a string or JSON
+as you would normally.
+
+> Note: When you deserialize a factor and want to use it for solving, you must call IncrementalInference.rebuildFactorMetadata!(dfgLoadInto, factor) to reinflate it completely. Please review [FileDFG service](src/FileDFG/services/FileDFG.jl) for an example.
+
+For example:
+```julia
+using DistributedFactorGraphs
+using IncrementalInference, RoME
+
+# Make a variable and a factor:
+# Make a simple graph
+dfg = GraphsDFG{SolverParams}(params=SolverParams())
+# Add the first pose :x0
+x0 = addVariable!(dfg, :x0, Pose2)
+# Add at a fixed location PriorPose2 to pin :x0 to a starting location (10,10, pi/4)
+prior = addFactor!(dfg, [:x0], PriorPose2( MvNormal([10; 10; 1.0/8.0], Matrix(Diagonal([0.1;0.1;0.05].^2))) ) )
+
+# Now serialize them:
+pVariable = packVariable(dfg, x0)
+pFactor = packFactor(dfg, prior)
+
+# And we can deserialize them
+upVariable = unpackVariable(dfg, pVariable)
+# FYI: The graph is used in unpackFactor to find the variables that the factor links to.
+upFactor = unpackFactor(dfg, pFactor, IncrementalInference)
+# Note, you need to call IncrementalInference.rebuildFactorMetadata!(dfgLoadInto, factor)
+# to make it useable. Please add an issue if this poses a problem or causes issues.
+```
+
+As a more complex example, we can use JSON2 to stringify the data and write it to a folder of files as FileDFG does:
+
+```julia
+using DistributedFactorGraphs
+using IncrementalInference, RoME
+
+# Make a variable and a factor:
+# Make a simple graph
+dfg = GraphsDFG{SolverParams}(params=SolverParams())
+# Add the first pose :x0
+x0 = addVariable!(dfg, :x0, Pose2)
+# Add at a fixed location PriorPose2 to pin :x0 to a starting location (10,10, pi/4)
+prior = addFactor!(dfg, [:x0], PriorPose2( MvNormal([10; 10; 1.0/8.0], Matrix(Diagonal([0.1;0.1;0.05].^2))) ) )
+
+# Slightly fancier example: We can use JSON2, we can serialize to a string
+varFolder = "/tmp"
+for v in getVariables(dfg)
+    vPacked = packVariable(dfg, v)
+    io = open("$varFolder/$(v.label).json", "w")
+    JSON2.write(io, vPacked)
+    close(io)
+end
+# Factors
+for f in getFactors(dfg)
+    fPacked = packFactor(dfg, f)
+    io = open("$folder/factors/$(f.label).json", "w")
+    JSON2.write(io, fPacked)
+    close(io)
+end
+```
diff --git a/src/CloudGraphsDFG/services/CloudGraphsDFG.jl b/src/CloudGraphsDFG/services/CloudGraphsDFG.jl
@@ -718,25 +718,11 @@ Optionally provide a distance to specify the number of edges should be followed.
 Optionally provide an existing subgraph addToDFG, the extracted nodes will be copied into this graph. By default a new subgraph will be created.
 Note: By default orphaned factors (where the subgraph does not contain all the related variables) are not returned. Set includeOrphanFactors to return the orphans irrespective of whether the subgraph contains all the variables.
 """
-function getSubgraphAroundNode(dfg::CloudGraphsDFG, node::DFGNode, distance::Int64=1, includeOrphanFactors::Bool=false, addToDFG::Union{Nothing, CloudGraphsDFG}=nothing)::CloudGraphsDFG
+function getSubgraphAroundNode(dfg::CloudGraphsDFG, node::DFGNode, distance::Int64=1, includeOrphanFactors::Bool=false, addToDFG::AbstractDFG=_getDuplicatedEmptyDFG(dfg))::AbstractDFG
     distance < 1 && error("getSubgraphAroundNode() only works for distance > 0")
 
     # Making a copy session if not specified
-    if addToDFG == nothing
-        addToDFG = _getDuplicatedEmptyDFG(dfg)
-    end
-
-    # Thank you Neo4j for 0..* awesomeness!!
-    neighborList = _getLabelsFromCyphonQuery(dfg.neo4jInstance, "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(node.label))-[FACTORGRAPH*0..$distance]-(node:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))")
-
-    # Copy the section of graph we want
-    _copyIntoGraph!(dfg, addToDFG, neighborList, includeOrphanFactors)
-    return addToDFG
-end
-
-function getSubgraphAroundNode(dfg::CloudGraphsDFG{<:AbstractParams}, node::DFGNode, distance::Int64, includeOrphanFactors::Bool, addToDFG::MetaGraphsDFG{AbstractParams})::AbstractDFG
-    distance < 1 && error("getSubgraphAroundNode() only works for distance > 0")
-
+    #moved to parameter addToDFG::AbstractDFG=_getDuplicatedEmptyDFG(dfg)
 
     # Thank you Neo4j for 0..* awesomeness!!
     neighborList = _getLabelsFromCyphonQuery(dfg.neo4jInstance, "(n:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId):$(node.label))-[FACTORGRAPH*0..$distance]-(node:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))")
@@ -801,6 +787,33 @@ function getAdjacencyMatrix(dfg::CloudGraphsDFG)::Matrix{Union{Nothing, Symbol}}
     return adjMat
 end
 
+function getAdjacencyMatrixSparse(dfg::CloudGraphsDFG)::Tuple{SparseMatrixCSC, Vector{Symbol}, Vector{Symbol}}
+    varLabels = getVariableIds(dfg)
+    factLabels = getFactorIds(dfg)
+    vDict = Dict(varLabels .=> [1:length(varLabels)...])
+    fDict = Dict(factLabels .=> [1:length(factLabels)...])
+
+    adjMat = spzeros(Int, length(factLabels), length(varLabels))
+
+    # Now ask for all relationships for this session graph
+    loadtx = transaction(dfg.neo4jInstance.connection)
+    query = "START n=node(*) MATCH (n:VARIABLE:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId))-[r:FACTORGRAPH]-(m:FACTOR:$(dfg.userId):$(dfg.robotId):$(dfg.sessionId)) RETURN n.label as variable, m.label as factor;"
+    nodes = loadtx(query; submit=true)
+    # Have to finish the transaction
+    commit(loadtx)
+    if length(nodes.errors) > 0
+        error(string(nodes.errors))
+    end
+    # Add in the relationships
+    varRel = Symbol.(map(node -> node["row"][1], nodes.results[1]["data"]))
+    factRel = Symbol.(map(node -> node["row"][2], nodes.results[1]["data"]))
+    for i = 1:length(varRel)
+        adjMat[fDict[factRel[i]], vDict[varRel[i]]] = 1
+    end
+
+    return adjMat, varLabels, factLabels
+end
+
 # """
 #     $(SIGNATURES)
 # Produces a dot-format of the graph for visualization.

diff --git a/src/Common.jl b/src/Common.jl
@@ -1,4 +1,5 @@
-export sortVarNested
+
+export sortVarNested, sortDFG
 export isPrior, lsfPriors
 export getVariableType, getSofttype
 export getFactorType, getfnctype
@@ -88,6 +89,24 @@ function sortVarNested(vars::Vector{Symbol})::Vector{Symbol}
 	return retvars
 end
 
+"""
+    $SIGNATURES
+
+Sort variable (factor) lists in a meaningful way, for example `[:april;:x1_3;:x1_6;]`
+
+Notes
+- Not fool proof, but does better than native sort.
+
+Example
+
+`sortDFG(ls(dfg))`
+
+Related
+
+ls, lsf
+"""
+sortDFG(vars::Vector{Symbol})::Vector{Symbol} = sortVarNested(vars)
+
 """
     $SIGNATURES
 

diff --git a/src/DFGPlots/DFGPlots.jl b/src/DFGPlots/DFGPlots.jl
@@ -4,6 +4,7 @@ using Colors
 using LightGraphs
 using MetaGraphs
 using GraphPlot
+using DocStringExtensions
 import GraphPlot: gplot
 
 using ...DistributedFactorGraphs
@@ -28,6 +29,26 @@ DFGPlotProps() = DFGPlotProps(  (var=colorant"seagreen", fac=colorant"cyan3"),
                                 true)
 
 
+
+"""
+    $(SIGNATURES)
+Plots the structure of the factor graph. GraphPlot must be imported before DistributedFactorGraphs for these functions to be available.
+Returns the plot context.
+
+E.g.
+```
+using GraphPlot
+using DistributedFactorGraphs, DistributedFactorGraphs.DFGPlots
+# ... Make graph...
+# Using GraphViz plotting
+dfgplot(fg)
+# Save to PDFG
+using Compose
+draw(PDF("/tmp/graph.pdf", 16cm, 16cm), dfgplot(fg))
+```
+
+More information at [GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl)
+"""
 function dfgplot(dfg::LightDFG, p::DFGPlotProps = DFGPlotProps())
 
     nodetypes = [haskey(dfg.g.variables, s) for s in dfg.g.labels]
@@ -46,8 +67,27 @@ function dfgplot(dfg::LightDFG, p::DFGPlotProps = DFGPlotProps())
 
 end
 
-function dfgplot(dfg::MetaGraphsDFG, p::DFGPlotProps = DFGPlotProps())
+"""
+    $(SIGNATURES)
+Plots the structure of the factor graph. GraphPlot must be imported before DistributedFactoGraphs for these functions to be available.
+Returns the plot context.
 
+E.g.
+```
+using GraphPlot
+using DistributedFactorGraphs, DistributedFactorGraphs.DFGPlots
+# ... Make graph...
+# Using GraphViz plotting
+dfgplot(fg)
+# Save to PDFG
+using Compose
+draw(PDF("/tmp/graph.pdf", 16cm, 16cm), dfgplot(fg))
+```
+
+More information at [GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl)
+"""
+function dfgplot(dfg::DistributedFactorGraphs.MetaGraphsDFG, p::DFGPlotProps = DFGPlotProps())
+    @info "Deprecated, please use GraphsDFG or LightDFG."
     nodesize = [has_prop(dfg.g,i,:factor) ?  p.nodesize.fac : p.nodesize.var for i=vertices(dfg.g)]
     if p.drawlabels
         nodelabel = [has_prop(dfg.g,i,:factor) ? "" : string(get_prop(dfg.g,i,:label)) for i=vertices(dfg.g)]
@@ -60,17 +100,36 @@ function dfgplot(dfg::MetaGraphsDFG, p::DFGPlotProps = DFGPlotProps())
 
 end
 
+"""
+    $(SIGNATURES)
+Plots the structure of the factor graph. GraphPlot must be imported before DistributedFactoGraphs for these functions to be available.
+Returns the plot context.
+
+E.g.
+```
+using GraphPlot
+using DistributedFactorGraphs, DistributedFactorGraphs.DFGPlots
+# ... Make graph...
+# Using GraphViz plotting
+dfgplot(fg)
+# Save to PDFG
+using Compose
+draw(PDF("/tmp/graph.pdf", 16cm, 16cm), dfgplot(fg))
+```
+
+More information at [GraphPlot.jl](https://github.com/JuliaGraphs/GraphPlot.jl)
+"""
 function dfgplot(dfg::AbstractDFG, p::DFGPlotProps = DFGPlotProps())
     # TODO implement convert functions
     @warn "TODO Implement convert"
-    ldfg = MetaGraphsDFG{AbstractParams}()
+    ldfg = LightDFG{AbstractParams}()
     DistributedFactorGraphs._copyIntoGraph!(dfg, ldfg, union(getVariableIds(dfg), getFactorIds(dfg)), true)
 
     dfgplot(ldfg, p)
-
 end
 
-function gplot(dfg::MetaGraphsDFG; keyargs...)
+function gplot(dfg::DistributedFactorGraphs.MetaGraphsDFG; keyargs...)
+    @info "Deprecated, please use GraphsDFG or LightDFG."
     gplot(dfg.g; keyargs...)
 end
 

diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl
@@ -11,23 +11,18 @@ using LinearAlgebra
 using SparseArrays
 
 # Entities
-include("entities/AbstractTypes.jl")
+include("entities/AbstractDFG.jl")
 include("entities/DFGFactor.jl")
 include("entities/DFGVariable.jl")
+include("entities/AbstractDFGSummary.jl")
 
 export AbstractDFG
 export AbstractParams, NoSolverParams
-export DFGNode
-
-export DFGFactor
+export DFGNode, DFGVariable, DFGFactor
 export InferenceType, PackedInferenceType, FunctorInferenceType, InferenceVariable, ConvolutionObject
-
 export FunctorSingleton, FunctorPairwise, FunctorPairwiseMinimize
-
-export DFGVariable
-export label, timestamp, tags, estimates, estimate, solverData, getData, solverDataDict, internalId, smallData, bigData
-export setSolverData
-export label, data, id
+export label, timestamp, tags, estimates, estimate, data, solverData, getData, solverDataDict, setSolverData, internalId, smallData, bigData
+export DFGVariableSummary, DFGFactorSummary, AbstractDFGSummary
 
 # Services/AbstractDFG Exports
 export hasFactor, hasVariable, isInitialized, getFactorFunction, isVariable, isFactor
@@ -38,13 +33,23 @@ export VariableNodeData, PackedVariableNodeData, VariableEstimate
 export GenericFunctionNodeData#, FunctionNodeData
 export getSerializationModule, setSerializationModule!
 export pack, unpack
+# Resolve with above
+export packVariable, unpackVariable, packFactor, unpackFactor
+
 
 #Interfaces
 export getAdjacencyMatrixSparse
 
+# File import and export
+export saveDFG, loadDFG
+
+# Summary functions
+export getSummary, getSummaryGraph
+
 # Common includes
 include("services/AbstractDFG.jl")
 include("services/DFGVariable.jl")
+include("services/DFGFactor.jl")
 
 # Include the Graphs.jl API.
 include("GraphsDFG/GraphsDFG.jl")
@@ -56,13 +61,11 @@ include("FileDFG/FileDFG.jl")
 include("MetaGraphsDFG/MetaGraphsDFG.jl")
 
 include("SymbolDFG/SymbolDFG.jl")
-@reexport using .SymbolDFGs
+using .SymbolDFGs
 
 include("LightDFG/LightDFG.jl")
 @reexport using .LightDFGs
 
-export saveDFG, loadDFG
-
 function __init__()
     @require Neo4j="d2adbeaf-5838-5367-8a2f-e46d570981db" begin
         # Include the Cloudgraphs API