diff --git a/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/DataMouseListener.java b/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/DataMouseListener.java
index d95c2ffb0..a44710e06 100644
--- a/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/DataMouseListener.java
+++ b/src/main/java/org/openstreetmap/josm/plugins/mapillary/gui/DataMouseListener.java
@@ -145,7 +145,7 @@ public void mouseMoved(MouseEvent e) {
                             layer.getData().setHighlighted(
                                 ways.stream().map(IPrimitive::getPrimitiveId).collect(Collectors.toSet()));
                             layer.invalidate();
-                        } else {
+                        } else if (!layer.getData().getHighlighted().isEmpty()) {
                             layer.getData().setHighlighted(Collections.emptyList());
                         }
                     } finally {