diff --git a/gtsam/hybrid/HybridGaussianFactorGraph.cpp b/gtsam/hybrid/HybridGaussianFactorGraph.cpp
index 049e6c38d2..b66847ca7a 100644
--- a/gtsam/hybrid/HybridGaussianFactorGraph.cpp
+++ b/gtsam/hybrid/HybridGaussianFactorGraph.cpp
@@ -584,9 +584,9 @@ GaussianFactorGraph HybridGaussianFactorGraph::choose(
 DiscreteFactorGraph HybridGaussianFactorGraph::discreteFactors() const {
   DiscreteFactorGraph dfg;
   for (auto &&f : factors_) {
-    auto discreteFactor = std::dynamic_pointer_cast<DiscreteFactor>(f);
-    assert(discreteFactor);
-    dfg.push_back(discreteFactor);
+    if (auto discreteFactor = std::dynamic_pointer_cast<DiscreteFactor>(f)) {
+      dfg.push_back(discreteFactor);
+    }
   }
   return dfg;
 }