diff --git a/object/ctx_rel.go b/object/ctx_rel.go
index 0c3c3fe..f21ceb9 100644
--- a/object/ctx_rel.go
+++ b/object/ctx_rel.go
@@ -3,7 +3,6 @@ package object
 import (
 	"fmt"
 
-	"github.com/dominikbraun/graph"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -16,26 +15,14 @@ func (sc *SourceContext) RefsByFileName(fileName string) ([]*RelVertex, error) {
 
 	// collect all the nodes starting from this file
 	startPoints := make([]*RelVertex, 0)
-	err := graph.BFS(sc.FactGraph, fileId, func(i int) bool {
-		// exclude itself
-		if i == fileId {
-			return false
-		}
-		if _, err := sc.FactGraph.Edge(fileId, i); err != nil {
-			return true
-		}
-
-		v, err := sc.FactGraph.Vertex(i)
+	for each := range sc.FactAdjMap[fileId] {
+		factVertex, err := sc.FactGraph.Vertex(each)
 		if err != nil {
-			log.Warnf("unknown vertex: %d", i)
-			return false
+			return nil, err
 		}
-		startPoints = append(startPoints, v.ToRelVertex())
-		return false
-	})
-	if err != nil {
-		return nil, err
+		startPoints = append(startPoints, factVertex.ToRelVertex())
 	}
+
 	return startPoints, nil
 }
 
@@ -84,27 +71,14 @@ func (sc *SourceContext) RefsFromDefId(defId int) ([]*FactVertex, error) {
 		return ret, nil
 	}
 
-	err = graph.BFS(sc.RelGraph, defId, func(i int) bool {
-		// exclude itself
-		if defId == i {
-			return false
-		}
-		// connected to current?
-		if _, err := sc.RelGraph.Edge(defId, i); err != nil {
-			return true
-		}
-
-		vertex, err := sc.FactGraph.Vertex(i)
+	for each := range sc.RelAdjMap[defId] {
+		vertex, err := sc.FactGraph.Vertex(each)
 		if err != nil {
-			return false
+			return nil, err
 		}
-
 		ret = append(ret, vertex)
-		return false
-	})
-	if err != nil {
-		return nil, err
 	}
+
 	return ret, nil
 }
 
diff --git a/object/source_context.go b/object/source_context.go
index 2d9280f..4a5eebd 100644
--- a/object/source_context.go
+++ b/object/source_context.go
@@ -93,6 +93,10 @@ type SourceContext struct {
 	FileMapping map[string]int
 	FactGraph   graph.Graph[int, *FactVertex]
 	RelGraph    graph.Graph[int, *RelVertex]
+
+	// caches
+	FactAdjMap map[int]map[int]graph.Edge[int]
+	RelAdjMap  map[int]map[int]graph.Edge[int]
 }
 
 func NewSourceContext() SourceContext {
@@ -103,5 +107,7 @@ func NewSourceContext() SourceContext {
 		FileMapping: make(map[string]int),
 		FactGraph:   factGraph,
 		RelGraph:    relGraph,
+		FactAdjMap:  nil,
+		RelAdjMap:   nil,
 	}
 }
diff --git a/parser/api.go b/parser/api.go
index aa1982e..d578ec8 100644
--- a/parser/api.go
+++ b/parser/api.go
@@ -288,6 +288,13 @@ func FromParser(readyParser *lsif.Parser) (*object.SourceContext, error) {
 		}
 	}
 
+	// cache for perf
+	// https://github.com/williamfzc/srctx/issues/44
+	factAdjMap, _ := factGraph.AdjacencyMap()
+	relAdjMap, _ := relGraph.AdjacencyMap()
+	ret.FactAdjMap = factAdjMap
+	ret.RelAdjMap = relAdjMap
+
 	factSize, err := factGraph.Size()
 	if err != nil {
 		return nil, err