From 19ad5a8ee05e88342874234f8cb720c9374ac49f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=86=A1=ED=98=84=EC=84=B1?= <129783824+thdgustjd1@users.noreply.github.com> Date: Tue, 25 Feb 2025 00:16:00 +0900 Subject: [PATCH] =?UTF-8?q?[UNI-205]=20refactor=20:=20SSE=20=EC=97=B0?= =?UTF-8?q?=EA=B2=B0=EC=9D=98=20=EC=95=88=EC=A0=95=EC=84=B1=EC=9D=B4=20?= =?UTF-8?q?=EB=B6=88=EC=95=88=EC=A0=95=ED=95=A0=EB=95=8C=20=EB=8C=80?= =?UTF-8?q?=EC=9D=91=ED=95=98=EA=B8=B0=20=EC=9C=84=ED=95=9C=20Stream=20?= =?UTF-8?q?=EB=B0=A9=EC=8B=9D=20API=20=EA=B0=9C=EB=B0=9C=20(#230)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [UNI-205] feat : Stream + 캐시 로직 작성 * [UNI-205] GetAllRoutesResInfo -> AllRoutesInfo로 변경 * 테스트용 CD * 테스트용 CD 삭제 * [UNI-205] feat : 캐시가 지워지지 않던 문제 해결 * cd 삭제 --- .../common/constant/UniroConst.java | 2 +- .../external/redis/RedisService.java | 4 +- .../uniro_backend/map/controller/MapApi.java | 9 +- .../map/controller/MapController.java | 17 ++- .../uniro_backend/map/service/MapService.java | 104 +++++++++++++++--- 5 files changed, 112 insertions(+), 24 deletions(-) diff --git a/uniro_backend/src/main/java/com/softeer5/uniro_backend/common/constant/UniroConst.java b/uniro_backend/src/main/java/com/softeer5/uniro_backend/common/constant/UniroConst.java index 9f00e5ee..304202e2 100644 --- a/uniro_backend/src/main/java/com/softeer5/uniro_backend/common/constant/UniroConst.java +++ b/uniro_backend/src/main/java/com/softeer5/uniro_backend/common/constant/UniroConst.java @@ -17,7 +17,7 @@ public final class UniroConst { public static final int CREATE_ROUTE_LIMIT_COUNT = 2000; - + public static final int MAX_CACHE_SIZE = 20; public static final Integer MAX_GOOGLE_API_BATCH_SIZE = 300; public static final String SUCCESS_STATUS = "OK"; } diff --git a/uniro_backend/src/main/java/com/softeer5/uniro_backend/external/redis/RedisService.java b/uniro_backend/src/main/java/com/softeer5/uniro_backend/external/redis/RedisService.java index e58947d6..e771f223 100644 --- a/uniro_backend/src/main/java/com/softeer5/uniro_backend/external/redis/RedisService.java +++ b/uniro_backend/src/main/java/com/softeer5/uniro_backend/external/redis/RedisService.java @@ -9,6 +9,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import static com.softeer5.uniro_backend.common.constant.UniroConst.MAX_CACHE_SIZE; + @Component @RequiredArgsConstructor @Slf4j @@ -39,7 +41,7 @@ public boolean hasData(String key){ public void deleteRoutesData(String key, int batchNumber) { String redisKeyPrefix = key + ":"; - for(int i=1; i<=batchNumber; i++){ + for(int i=1; i<= Math.max(MAX_CACHE_SIZE,batchNumber); i++){ redisTemplate.delete(redisKeyPrefix + i); } } diff --git a/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/controller/MapApi.java b/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/controller/MapApi.java index fb577147..14ba73a4 100644 --- a/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/controller/MapApi.java +++ b/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/controller/MapApi.java @@ -28,7 +28,14 @@ public interface MapApi { @ApiResponse(responseCode = "200", description = "모든 지도 조회 성공"), @ApiResponse(responseCode = "400", description = "EXCEPTION(임시)", content = @Content), }) - ResponseEntity getAllRoutesAndNodes(@PathVariable("univId") Long univId); + ResponseEntity getAllRoutesAndNodes(@PathVariable("univId") Long univId); + + @Operation(summary = "모든 지도(노드,루트) 조회 by stream") + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "모든 지도 조회 성공"), + @ApiResponse(responseCode = "400", description = "EXCEPTION(임시)", content = @Content), + }) + ResponseEntity getAllRoutesAndNodesStream(@PathVariable("univId") Long univId); @Operation(summary = "모든 지도(노드,루트) 조회 by sse") @ApiResponses(value = { diff --git a/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/controller/MapController.java b/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/controller/MapController.java index 81d07900..da7bbfb0 100644 --- a/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/controller/MapController.java +++ b/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/controller/MapController.java @@ -26,15 +26,22 @@ public class MapController implements MapApi { private final AdminService adminService; @GetMapping("/{univId}/routes-local") - public ResponseEntity getAllRoutesAndNodesByLocalCache(@PathVariable("univId") Long univId){ - GetAllRoutesResDTO allRoutes = mapService.getAllRoutesByLocalCache(univId); + public ResponseEntity getAllRoutesAndNodesByLocalCache(@PathVariable("univId") Long univId){ + AllRoutesInfo allRoutes = mapService.getAllRoutesByLocalCache(univId); return ResponseEntity.ok().body(allRoutes); } @Override @GetMapping("/{univId}/routes") - public ResponseEntity getAllRoutesAndNodes(@PathVariable("univId") Long univId){ - GetAllRoutesResDTO allRoutes = mapService.getAllRoutes(univId); + public ResponseEntity getAllRoutesAndNodes(@PathVariable("univId") Long univId){ + AllRoutesInfo allRoutes = mapService.getAllRoutes(univId); + return ResponseEntity.ok().body(allRoutes); + } + + @Override + @GetMapping("/{univId}/routes/stream") + public ResponseEntity getAllRoutesAndNodesStream(@PathVariable("univId") Long univId){ + AllRoutesInfo allRoutes = mapService.getAllRoutesByStream(univId); return ResponseEntity.ok().body(allRoutes); } @@ -42,7 +49,7 @@ public ResponseEntity getAllRoutesAndNodes(@PathVariable("un @GetMapping("/{univId}/routes/sse") public SseEmitter getAllRoutes(@PathVariable("univId") Long univId){ SseEmitter emitter = new SseEmitter(60 * 1000L); // timeout 1분 - mapService.getAllRoutesByStream(univId, emitter); + mapService.getAllRoutesBySSE(univId, emitter); return emitter; } diff --git a/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/service/MapService.java b/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/service/MapService.java index c0f94d4e..416e5959 100644 --- a/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/service/MapService.java +++ b/uniro_backend/src/main/java/com/softeer5/uniro_backend/map/service/MapService.java @@ -63,7 +63,7 @@ public class MapService { private final Map> cache = new HashMap<>(); - public GetAllRoutesResDTO getAllRoutesByLocalCache(Long univId) { + public AllRoutesInfo getAllRoutesByLocalCache(Long univId) { if(!cache.containsKey(univId)){ List routes = routeRepository.findAllRouteByUnivIdWithNodes(univId); @@ -78,17 +78,10 @@ public GetAllRoutesResDTO getAllRoutesByLocalCache(Long univId) { throw new RouteException("Route Not Found", ROUTE_NOT_FOUND); } - RevInfo revInfo = revInfoRepository.findFirstByUnivIdOrderByRevDesc(univId) - .orElseThrow(() -> new RouteException("Revision not found", RECENT_REVISION_NOT_FOUND)); - - AllRoutesInfo allRoutesInfo = routeCacheCalculator.assembleRoutes(routes); - GetAllRoutesResDTO response = GetAllRoutesResDTO.of(allRoutesInfo.getNodeInfos(), allRoutesInfo.getCoreRoutes(), - allRoutesInfo.getBuildingRoutes(), revInfo.getRev()); - - return response; + return routeCacheCalculator.assembleRoutes(routes); } - public GetAllRoutesResDTO getAllRoutes(Long univId) { + public AllRoutesInfo getAllRoutes(Long univId) { if(!redisService.hasData(univId.toString())){ List routes = routeRepository.findAllRouteByUnivIdWithNodes(univId); @@ -108,17 +101,96 @@ public GetAllRoutesResDTO getAllRoutes(Long univId) { throw new RouteException("Route Not Found", ROUTE_NOT_FOUND); } - RevInfo revInfo = revInfoRepository.findFirstByUnivIdOrderByRevDesc(univId) - .orElseThrow(() -> new RouteException("Revision not found", RECENT_REVISION_NOT_FOUND)); + return routeCacheCalculator.assembleRoutes(routes); + } + + public AllRoutesInfo getAllRoutesByStream(Long univId) { + List nodeInfos = new ArrayList<>(); + List coreRoutes = new ArrayList<>(); + List buildingRoutes = new ArrayList<>(); + + String redisKeyPrefix = univId + ":"; + int batchNumber = 1; + + if (!processRedisDataByStream(redisKeyPrefix, batchNumber, nodeInfos, coreRoutes, buildingRoutes)) { + processDatabaseDataByStream(univId, redisKeyPrefix, batchNumber, nodeInfos, coreRoutes, buildingRoutes); + } + + return AllRoutesInfo.of(nodeInfos, coreRoutes, buildingRoutes); + } + + private boolean processRedisDataByStream(String redisKeyPrefix, + int batchNumber, + List nodeInfos, + List coreRoutes, + List buildingRoutes){ + while (redisService.hasData(redisKeyPrefix + batchNumber)) { + LightRoutes lightRoutes = (LightRoutes) redisService.getData(redisKeyPrefix + batchNumber); + if (lightRoutes == null) { + break; + } + + processBatchByStream(lightRoutes.getLightRoutes(), nodeInfos, coreRoutes, buildingRoutes); + batchNumber++; + } + + return batchNumber > 1; + } + + private void processDatabaseDataByStream(Long univId, String redisKeyPrefix, int batchNumber, + List nodeInfos, + List coreRoutes, + List buildingRoutes) { + int fetchSize = routeRepository.countByUnivId(univId); + int remain = fetchSize%STREAM_FETCH_SIZE; + fetchSize = fetchSize/STREAM_FETCH_SIZE + (remain > 0 ? 1 : 0); + + try (Stream routeStream = routeRepository.findAllLightRoutesByUnivId(univId)) { + List batch = new ArrayList<>(STREAM_FETCH_SIZE); + for (LightRoute route : (Iterable) routeStream::iterator) { + batch.add(route); + + if (batch.size() == STREAM_FETCH_SIZE) { + saveAndSendBatchByStream(redisKeyPrefix, batchNumber++, batch, nodeInfos, coreRoutes, buildingRoutes); + } + } + + // 남은 배치 처리 + if (!batch.isEmpty()) { + saveAndSendBatchByStream(redisKeyPrefix, batchNumber, batch, nodeInfos, coreRoutes, buildingRoutes); + } + + redisService.saveDataToString(univId.toString() + ":fetch", String.valueOf(fetchSize)); + } + + } - AllRoutesInfo allRoutesInfo = routeCacheCalculator.assembleRoutes(routes); + private void saveAndSendBatchByStream(String redisKeyPrefix, int batchNumber, List batch, + List nodeInfos, + List coreRoutes, + List buildingRoutes) { + LightRoutes value = new LightRoutes(batch); + redisService.saveData(redisKeyPrefix + batchNumber, value); + processBatchByStream(batch, nodeInfos, coreRoutes, buildingRoutes); + batch.clear(); + } - return GetAllRoutesResDTO.of(allRoutesInfo.getNodeInfos(), allRoutesInfo.getCoreRoutes(), - allRoutesInfo.getBuildingRoutes(), revInfo.getRev()); + private void processBatchByStream(List batch, + List nodeInfos, + List coreRoutes, + List buildingRoutes) { + if (!batch.isEmpty()) { + AllRoutesInfo allRoutesInfo = routeCacheCalculator.assembleRoutes(batch); + nodeInfos.addAll(allRoutesInfo.getNodeInfos()); + coreRoutes.addAll(allRoutesInfo.getCoreRoutes()); + buildingRoutes.addAll(allRoutesInfo.getBuildingRoutes()); + batch.clear(); + entityManager.clear(); + } } @Async - public void getAllRoutesByStream(Long univId, SseEmitter emitter) { + public void getAllRoutesBySSE(Long univId, SseEmitter emitter) { String redisKeyPrefix = univId + ":"; int batchNumber = 1;