From 92f3c74db13989f01b0c46e2a4c5308266024d31 Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 1 Aug 2023 18:36:44 +0200 Subject: [PATCH 1/3] prepare blob lazy-loading --- cmd/explorer/main.go | 1 + config/default.config.yml | 1 + handlers/pageData.go | 12 ++----- handlers/slot.go | 66 +++++++++++++++++++++++++++------------ services/beaconservice.go | 4 +++ templates/slot/blobs.html | 34 ++++++++++++-------- templates/slot/slot.html | 28 ++++++++++++----- test-config.yaml | 1 + types/models/slot.go | 10 +++++- 9 files changed, 105 insertions(+), 52 deletions(-) diff --git a/cmd/explorer/main.go b/cmd/explorer/main.go index c3e86a3c..c43a1a5d 100644 --- a/cmd/explorer/main.go +++ b/cmd/explorer/main.go @@ -69,6 +69,7 @@ func startFrontend() { router.HandleFunc("/epoch/{epoch}", handlers.Epoch).Methods("GET") router.HandleFunc("/slots", handlers.Slots).Methods("GET") router.HandleFunc("/slot/{slotOrHash}", handlers.Slot).Methods("GET") + router.HandleFunc("/slot/{hash}/blob/{blobIdx}", handlers.SlotBlob).Methods("GET") router.HandleFunc("/search", handlers.Search).Methods("GET") router.HandleFunc("/search/{type}", handlers.SearchAhead).Methods("GET") router.HandleFunc("/validators", handlers.Validators).Methods("GET") diff --git a/config/default.config.yml b/config/default.config.yml index 7502f3e4..c7798d04 100644 --- a/config/default.config.yml +++ b/config/default.config.yml @@ -14,6 +14,7 @@ server: frontend: enabled: true # Enable or disable to web frontend debug: false + minimize: false # minimize html templates # Name of the site, displayed in the title tag siteName: "Beaconchain Light" diff --git a/handlers/pageData.go b/handlers/pageData.go index 1eae4a05..b9f4813c 100644 --- a/handlers/pageData.go +++ b/handlers/pageData.go @@ -21,10 +21,10 @@ var layoutTemplateFiles = []string{ } func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title string, mainTemplates []string) *types.PageData { - fullTitle := fmt.Sprintf("%v - %v - beaconchain light - %v", title, utils.Config.Frontend.SiteName, time.Now().Year()) + fullTitle := fmt.Sprintf("%v - %v - %v", title, utils.Config.Frontend.SiteName, time.Now().Year()) if title == "" { - fullTitle = fmt.Sprintf("%v - beaconchain light - %v", utils.Config.Frontend.SiteName, time.Now().Year()) + fullTitle = fmt.Sprintf("%v - %v", utils.Config.Frontend.SiteName, time.Now().Year()) } isMainnet := utils.Config.Chain.Config.ConfigName == "mainnet" @@ -70,14 +70,6 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st return data } -func SetPageDataTitle(pageData *types.PageData, title string) { - if title == "" { - pageData.Meta.Title = fmt.Sprintf("%v - light beaconchain explorer - %v", utils.Config.Frontend.SiteName, time.Now().Year()) - } else { - pageData.Meta.Title = fmt.Sprintf("%v - %v - light beaconchain explorer - %v", title, utils.Config.Frontend.SiteName, time.Now().Year()) - } -} - func createMenuItems(active string, isMain bool) []types.MainMenuItem { hiddenFor := []string{"confirmation", "login", "register"} diff --git a/handlers/slot.go b/handlers/slot.go index 58464c8f..42fddf99 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -2,6 +2,7 @@ package handlers import ( "encoding/hex" + "encoding/json" "fmt" "net/http" "strconv" @@ -70,23 +71,57 @@ func Slot(w http.ResponseWriter, r *http.Request) { } } +// SlotBlob handles responses for the block blobs tab +func SlotBlob(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + vars := mux.Vars(r) + blockRoot, err := hex.DecodeString(strings.Replace(vars["hash"], "0x", "", -1)) + if err != nil || len(blockRoot) != 32 { + http.Error(w, "Internal server error", http.StatusServiceUnavailable) + return + } + blobIdx, err := strconv.ParseUint(vars["blobIdx"], 10, 64) + if err != nil { + http.Error(w, "Internal server error", http.StatusServiceUnavailable) + return + } + blobData, err := services.GlobalBeaconService.GetBlobSidecarsByBlockRoot(blockRoot) + if err != nil { + logrus.WithError(err).Error("error loading blob sidecar") + http.Error(w, "Internal server error", http.StatusServiceUnavailable) + return + } + var result interface{} + if blobData != nil && blobIdx < uint64(len(blobData.Data)) { + result = blobData.Data[blobIdx] + } + err = json.NewEncoder(w).Encode(result) + if err != nil { + logrus.WithError(err).Error("error encoding blob sidecar") + http.Error(w, "Internal server error", http.StatusServiceUnavailable) + } +} + func getSlotPageData(blockSlot int64, blockRoot []byte) *models.SlotPageData { pageData := &models.SlotPageData{} pageCacheKey := fmt.Sprintf("slot:%v:%x", blockSlot, blockRoot) - pageData = services.GlobalFrontendCache.ProcessCachedPage(pageCacheKey, false, pageData, func(pageCall *services.FrontendCacheProcessingPage) interface{} { - return buildSlotPageData(blockSlot, blockRoot) + pageData = services.GlobalFrontendCache.ProcessCachedPage(pageCacheKey, true, pageData, func(pageCall *services.FrontendCacheProcessingPage) interface{} { + pageData, cacheTimeout := buildSlotPageData(blockSlot, blockRoot) + pageCall.CacheTimeout = cacheTimeout + return pageData }).(*models.SlotPageData) return pageData } -func buildSlotPageData(blockSlot int64, blockRoot []byte) *models.SlotPageData { +func buildSlotPageData(blockSlot int64, blockRoot []byte) (*models.SlotPageData, time.Duration) { finalizedHead, err := services.GlobalBeaconService.GetFinalizedBlockHead() var blockData *rpctypes.CombinedBlockResponse if err == nil { if blockSlot > -1 { - blockData, err = services.GlobalBeaconService.GetSlotDetailsBySlot(uint64(blockSlot), true) + blockData, err = services.GlobalBeaconService.GetSlotDetailsBySlot(uint64(blockSlot), false) } else { - blockData, err = services.GlobalBeaconService.GetSlotDetailsByBlockroot(blockRoot, true) + blockData, err = services.GlobalBeaconService.GetSlotDetailsByBlockroot(blockRoot, false) } } @@ -104,7 +139,7 @@ func buildSlotPageData(blockSlot int64, blockRoot []byte) *models.SlotPageData { } if blockData == nil { - return nil + return nil, -1 } slot := uint64(blockData.Header.Data.Header.Message.Slot) logrus.Printf("slot page called: %v", slot) @@ -142,7 +177,7 @@ func buildSlotPageData(blockSlot int64, blockRoot []byte) *models.SlotPageData { pageData.Block = getSlotPageBlockData(blockData, assignments) } - return pageData + return pageData, -1 } func getSlotPageBlockData(blockData *rpctypes.CombinedBlockResponse, assignments *rpctypes.EpochAssignments) *models.SlotPageBlockData { @@ -338,22 +373,13 @@ func getSlotPageBlockData(blockData *rpctypes.CombinedBlockResponse, assignments } } - if epoch >= utils.Config.Chain.Config.DenebForkEpoch && blockData.Blobs != nil { - pageData.BlobsCount = uint64(len(blockData.Blobs.Data)) + if epoch >= utils.Config.Chain.Config.DenebForkEpoch { + pageData.BlobsCount = uint64(len(blockData.Block.Data.Message.Body.BlobKzgCommitments)) pageData.Blobs = make([]*models.SlotPageBlob, pageData.BlobsCount) for i := uint64(0); i < pageData.BlobsCount; i++ { - blob := blockData.Blobs.Data[i] blobData := &models.SlotPageBlob{ - Index: uint64(blob.Index), - KzgCommitment: blob.KzgCommitment, - KzgProof: blob.KzgProof, - Blob: blob.Blob, - } - if len(blob.Blob) > 512 { - blobData.BlobShort = blob.Blob[0:512] - blobData.IsShort = true - } else { - blobData.BlobShort = blob.Blob + Index: i, + KzgCommitment: blockData.Block.Data.Message.Body.BlobKzgCommitments[i], } pageData.Blobs[i] = blobData } diff --git a/services/beaconservice.go b/services/beaconservice.go index 740c3a67..1af173a7 100644 --- a/services/beaconservice.go +++ b/services/beaconservice.go @@ -170,6 +170,10 @@ func (bs *BeaconService) GetSlotDetailsBySlot(slot uint64, withBlobs bool) (*rpc return result, nil } +func (bs *BeaconService) GetBlobSidecarsByBlockRoot(blockroot []byte) (*rpctypes.StandardV1BlobSidecarsResponse, error) { + return bs.rpcClient.GetBlobSidecarsByBlockroot(blockroot) +} + func (bs *BeaconService) GetOrphanedBlock(blockroot []byte) *rpctypes.CombinedBlockResponse { orphanedBlock := db.GetOrphanedBlock(blockroot) if orphanedBlock == nil { diff --git a/templates/slot/blobs.html b/templates/slot/blobs.html index 1e56a838..03964cb9 100644 --- a/templates/slot/blobs.html +++ b/templates/slot/blobs.html @@ -12,21 +12,29 @@ -
-
KGZ Proof:
-
- 0x{{ printf "%x" $blob.KzgProof }} - + {{ if $blob.HaveData }} +
+
KGZ Proof:
+
+ 0x{{ printf "%x" $blob.KzgProof }} + +
-
-
-
Data:
-
- 0x{{ printf "%x" $blob.BlobShort }} - {{- if $blob.IsShort -}}...{{ end }} - +
+
Data:
+
+ 0x{{ printf "%x" $blob.BlobShort }} + {{- if $blob.IsShort -}}...{{ end }} + +
-
+ {{ else }} +
+
+ load +
+
+ {{ end }}
{{ end }} diff --git a/templates/slot/slot.html b/templates/slot/slot.html index 212f334b..3f2b5c7a 100644 --- a/templates/slot/slot.html +++ b/templates/slot/slot.html @@ -76,7 +76,7 @@

{{ if .Block }} -
+
@@ -87,7 +87,7 @@

Showing {{ .Block.AttestationsCount }} A {{ template "block_attestations" . }}

{{ if gt .Block.DepositsCount 0 }} -
+
@@ -99,7 +99,7 @@

Showing {{ .Block.DepositsCount }} Depos

{{ end }} {{ if gt .Block.VoluntaryExitsCount 0 }} -
+
@@ -111,7 +111,7 @@

Showing {{ .Block.VoluntaryExitsCount }}

{{ end }} {{ if gt .Block.AttesterSlashingsCount 0 }} -
+
@@ -123,7 +123,7 @@

Showing {{ .Block.AttesterSlashingsCount

{{ end }} {{ if gt .Block.ProposerSlashingsCount 0 }} -
+
@@ -135,7 +135,7 @@

Showing {{ .Block.ProposerSlashingsCount

{{ end }} {{ if gt .Block.WithdrawalsCount 0 }} -
+
@@ -147,7 +147,7 @@

Showing {{ .Block.WithdrawalsCount }} Wi

{{ end }} {{ if gt .Block.BLSChangesCount 0 }} -
+
@@ -159,7 +159,7 @@

Showing {{ .Block.BLSChangesCount }} BLS

{{ end }} {{ if gt .Block.BlobsCount 0 }} -
+
@@ -173,6 +173,18 @@

Showing {{ .Block.BlobsCount }} Blob sid {{ end }}

+
{{ end }} diff --git a/test-config.yaml b/test-config.yaml index 9ac2cb43..2764368f 100644 --- a/test-config.yaml +++ b/test-config.yaml @@ -17,6 +17,7 @@ server: frontend: enabled: true # Enable or disable to web frontend debug: true + minimize: false siteName: "Beaconchain Light" # Name of the site, displayed in the title tag siteSubtitle: "Ephemery" ethExplorerLink: "https://explorer.ephemery.dev/" diff --git a/types/models/slot.go b/types/models/slot.go index e36b2738..e626c32b 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -171,8 +171,16 @@ type SlotPageWithdrawal struct { type SlotPageBlob struct { Index uint64 `json:"index"` - BlobShort []byte `json:"blob_short"` + KzgCommitment []byte `json:"kzg_commitment"` + HaveData bool `json:"have_data"` IsShort bool `json:"is_short"` + BlobShort []byte `json:"blob_short"` + Blob []byte `json:"blob"` + KzgProof []byte `json:"kzg_proof"` +} + +type SlotPageBlobDetails struct { + Index uint64 `json:"index"` Blob []byte `json:"blob"` KzgCommitment []byte `json:"kzg_commitment"` KzgProof []byte `json:"kzg_proof"` From 32d961644a168c330c55cd416e365defbec512ad Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 1 Aug 2023 19:11:01 +0200 Subject: [PATCH 2/3] lazy load blobs (js implementation) --- handlers/slot.go | 27 ++++++++++++++++++- static/js/explorer.js | 10 +++---- templates/slot/blobs.html | 56 ++++++++++++++++++++++++++++++++++++--- types/models/slot.go | 6 ++--- 4 files changed, 87 insertions(+), 12 deletions(-) diff --git a/handlers/slot.go b/handlers/slot.go index 42fddf99..4b42d76f 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -63,6 +63,25 @@ func Slot(w http.ResponseWriter, r *http.Request) { return } + urlArgs := r.URL.Query() + if urlArgs.Has("blob") && pageData.Block != nil { + blobData, err := services.GlobalBeaconService.GetBlobSidecarsByBlockRoot(pageData.Block.BlockRoot) + if err == nil && blobData != nil { + for blobIdx, blob := range blobData.Data { + blobData := pageData.Block.Blobs[blobIdx] + blobData.HaveData = true + blobData.KzgProof = blob.KzgProof + blobData.Blob = blob.Blob + if len(blob.Blob) > 512 { + blobData.BlobShort = blob.Blob[0:512] + blobData.IsShort = true + } else { + blobData.BlobShort = blob.Blob + } + } + } + } + template := templates.GetTemplate(slotTemplateFiles...) data := InitPageData(w, r, "blockchain", "/slots", fmt.Sprintf("Slot %v", slotOrHash), slotTemplateFiles) data.Data = pageData @@ -94,7 +113,13 @@ func SlotBlob(w http.ResponseWriter, r *http.Request) { } var result interface{} if blobData != nil && blobIdx < uint64(len(blobData.Data)) { - result = blobData.Data[blobIdx] + blob := blobData.Data[blobIdx] + result = &models.SlotPageBlobDetails{ + Index: blobIdx, + KzgCommitment: blob.KzgCommitment.String(), + KzgProof: blob.KzgProof.String(), + Blob: blob.Blob.String(), + } } err = json.NewEncoder(w).Encode(result) if err != nil { diff --git a/static/js/explorer.js b/static/js/explorer.js index c809885d..5eb95b30 100644 --- a/static/js/explorer.js +++ b/static/js/explorer.js @@ -2,11 +2,14 @@ (function() { window.addEventListener('DOMContentLoaded', function() { initControls(); + window.setInterval(updateTimers, 1000); + initHeaderSearch(); }); + window.explorer = { + initControls: initControls, + }; function initControls() { - window.setInterval(updateTimers, 1000); - // init tooltips var tooltipEls = document.querySelectorAll('[data-bs-toggle="tooltip"]'); Array.prototype.forEach.call(tooltipEls, function(tooltipEl) { @@ -33,9 +36,6 @@ tooltip.setContent({ '.tooltip-inner': title }); }, 1000); }); - - // init type-ahead search - initHeaderSearch() } function updateTimers() { diff --git a/templates/slot/blobs.html b/templates/slot/blobs.html index 03964cb9..1d68b254 100644 --- a/templates/slot/blobs.html +++ b/templates/slot/blobs.html @@ -29,13 +29,63 @@
{{ else }} -
-
- load +
+
+
+ Load Blob Data +
{{ end }}
{{ end }} + {{ end }} diff --git a/types/models/slot.go b/types/models/slot.go index e626c32b..5a85db52 100644 --- a/types/models/slot.go +++ b/types/models/slot.go @@ -181,7 +181,7 @@ type SlotPageBlob struct { type SlotPageBlobDetails struct { Index uint64 `json:"index"` - Blob []byte `json:"blob"` - KzgCommitment []byte `json:"kzg_commitment"` - KzgProof []byte `json:"kzg_proof"` + Blob string `json:"blob"` + KzgCommitment string `json:"kzg_commitment"` + KzgProof string `json:"kzg_proof"` } From c4f0ae145ec31edbb7aa578bd7fecfd3fb1b34ef Mon Sep 17 00:00:00 2001 From: pk910 Date: Tue, 1 Aug 2023 19:21:33 +0200 Subject: [PATCH 3/3] enable caching for base slot details --- handlers/slot.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/handlers/slot.go b/handlers/slot.go index 4b42d76f..db333fcf 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -140,6 +140,7 @@ func getSlotPageData(blockSlot int64, blockRoot []byte) *models.SlotPageData { } func buildSlotPageData(blockSlot int64, blockRoot []byte) (*models.SlotPageData, time.Duration) { + currentSlot := utils.TimeToSlot(uint64(time.Now().Unix())) finalizedHead, err := services.GlobalBeaconService.GetFinalizedBlockHead() var blockData *rpctypes.CombinedBlockResponse if err == nil { @@ -176,6 +177,7 @@ func buildSlotPageData(blockSlot int64, blockRoot []byte) (*models.SlotPageData, Ts: utils.SlotToTime(slot), NextSlot: slot + 1, PreviousSlot: slot - 1, + Future: slot >= currentSlot, } assignments, err := services.GlobalBeaconService.GetEpochAssignments(utils.EpochOfSlot(slot)) @@ -184,6 +186,22 @@ func buildSlotPageData(blockSlot int64, blockRoot []byte) (*models.SlotPageData, // we can safely continue here. the UI is prepared to work without epoch duties, but fields related to the duties are not shown } + var cacheTimeout time.Duration + if pageData.Future { + timeDiff := pageData.Ts.Sub(time.Now()) + if timeDiff > 10*time.Minute { + cacheTimeout = 10 * time.Minute + } else { + cacheTimeout = timeDiff + } + } else if pageData.EpochFinalized { + cacheTimeout = 10 * time.Minute + } else if blockData != nil { + cacheTimeout = 5 * time.Minute + } else { + cacheTimeout = 10 * time.Second + } + if blockData == nil { pageData.Status = uint16(models.SlotStatusMissed) @@ -202,7 +220,7 @@ func buildSlotPageData(blockSlot int64, blockRoot []byte) (*models.SlotPageData, pageData.Block = getSlotPageBlockData(blockData, assignments) } - return pageData, -1 + return pageData, cacheTimeout } func getSlotPageBlockData(blockData *rpctypes.CombinedBlockResponse, assignments *rpctypes.EpochAssignments) *models.SlotPageBlockData {