Skip to content

Commit

Permalink
refactor: clean up flags and configs (#146)
Browse files Browse the repository at this point in the history
* refactor: move flags from config.go -> flags.go

* refactor: move flags+config to new cli package

* refactor: stores into subdirs (precomputed_key and generated_key)

* fix: memstore test

* refactor: move config back to server package + rename cli->flags

* refactor: move redis flags+config into redis package cli.go

* refactor: move s3 flags to their own package cli.go

* refactor: move eigenda_client flags to their own (temporary) package

* refactor: move verifier flags/config into its own package cli.go

* fix: lint
Also added WaitForFinalization eigenda flag

* refactor: move memstore flags to their own package cli.go

* fix: e2e holesky-test

* fix: e2e tests

* fix: e2e tests (attempt 2)

* refactor: rename KeyGeneratedStore -> GeneratedKeyStore

* style(flags): remove unneeded explicit "" values

* style: add category to all s3 flags (forgot in previous commit)

* style: lint
  • Loading branch information
samlaf authored Sep 27, 2024
1 parent d4ef8c5 commit 2600bad
Show file tree
Hide file tree
Showing 24 changed files with 862 additions and 715 deletions.
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ LDFLAGSSTRING +=-X main.Date=$(BUILD_TIME)
LDFLAGSSTRING +=-X main.Version=$(GIT_TAG)
LDFLAGS := -ldflags "$(LDFLAGSSTRING)"

E2ETEST = INTEGRATION=true go test -timeout 1m -v ./e2e -parallel 4 -deploy-config ../.devnet/devnetL1.json
HOLESKYTEST = TESTNET=true go test -timeout 50m -v ./e2e -parallel 4 -deploy-config ../.devnet/devnetL1.json
E2ETEST = INTEGRATION=true go test -timeout 1m ./e2e -parallel 4 -deploy-config ../.devnet/devnetL1.json
HOLESKYTEST = TESTNET=true go test -timeout 50m ./e2e -parallel 4 -deploy-config ../.devnet/devnetL1.json

.PHONY: eigenda-proxy
eigenda-proxy:
Expand Down
6 changes: 3 additions & 3 deletions cmd/server/entrypoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"

"github.com/Layr-Labs/eigenda-proxy/flags"
"github.com/Layr-Labs/eigenda-proxy/metrics"
"github.com/Layr-Labs/eigenda-proxy/server"
"github.com/urfave/cli/v2"
Expand All @@ -24,15 +25,14 @@ func StartProxySvr(cliCtx *cli.Context) error {
ctx, ctxCancel := context.WithCancel(cliCtx.Context)
defer ctxCancel()

m := metrics.NewMetrics("default")

log.Info("Initializing EigenDA proxy server...")

daRouter, err := server.LoadStoreRouter(ctx, cfg, log)
if err != nil {
return fmt.Errorf("failed to create store: %w", err)
}
server := server.NewServer(cliCtx.String(server.ListenAddrFlagName), cliCtx.Int(server.PortFlagName), daRouter, log, m)
m := metrics.NewMetrics("default")
server := server.NewServer(cliCtx.String(flags.ListenAddrFlagName), cliCtx.Int(flags.PortFlagName), daRouter, log, m)

if err := server.Start(); err != nil {
return fmt.Errorf("failed to start the DA server: %w", err)
Expand Down
4 changes: 2 additions & 2 deletions cmd/server/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import (
"github.com/joho/godotenv"
"github.com/urfave/cli/v2"

"github.com/Layr-Labs/eigenda-proxy/flags"
"github.com/Layr-Labs/eigenda-proxy/metrics"
"github.com/Layr-Labs/eigenda-proxy/server"
"github.com/ethereum-optimism/optimism/op-service/cliapp"
"github.com/ethereum-optimism/optimism/op-service/ctxinterrupt"
oplog "github.com/ethereum-optimism/optimism/op-service/log"
Expand All @@ -26,7 +26,7 @@ func main() {
oplog.SetupDefaults()

app := cli.NewApp()
app.Flags = cliapp.ProtectFlags(server.Flags)
app.Flags = cliapp.ProtectFlags(flags.Flags)
app.Version = Version
app.Name = "eigenda-proxy"
app.Usage = "EigenDA Proxy Sidecar Service"
Expand Down
4 changes: 2 additions & 2 deletions e2e/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func TestKeccak256CommitmentRequestErrorsWhenS3NotSet(t *testing.T) {
testCfg.UseKeccak256ModeS3 = true

tsConfig := e2e.TestSuiteConfig(t, testCfg)
tsConfig.S3Config.Endpoint = ""
tsConfig.EigenDAConfig.S3Config.Endpoint = ""
ts, kill := e2e.CreateTestSuite(t, tsConfig)
defer kill()

Expand Down Expand Up @@ -393,7 +393,7 @@ func TestProxyServerCachingWithRedis(t *testing.T) {
require.Equal(t, testPreimage, preimage)

// ensure that read was from cache
redStats, err := ts.Server.GetStoreStats(store.Redis)
redStats, err := ts.Server.GetStoreStats(store.RedisBackendType)
require.NoError(t, err)

require.Equal(t, 1, redStats.Reads)
Expand Down
77 changes: 47 additions & 30 deletions e2e/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,19 @@ import (
"context"
"fmt"
"os"
"runtime"
"testing"
"time"

"github.com/Layr-Labs/eigenda-proxy/metrics"
"github.com/Layr-Labs/eigenda-proxy/server"
"github.com/Layr-Labs/eigenda-proxy/store"
"github.com/Layr-Labs/eigenda-proxy/store/generated_key/memstore"
"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
"github.com/Layr-Labs/eigenda-proxy/utils"
"github.com/Layr-Labs/eigenda-proxy/verify"
"github.com/Layr-Labs/eigenda/api/clients"
"github.com/Layr-Labs/eigenda/encoding/kzg"
"github.com/ethereum/go-ethereum/log"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
Expand Down Expand Up @@ -53,15 +59,15 @@ func TestConfig(useMemory bool) *Cfg {
}

func createRedisConfig(eigendaCfg server.Config) server.CLIConfig {
eigendaCfg.RedisConfig = redis.Config{
Endpoint: "127.0.0.1:9001",
Password: "",
DB: 0,
Eviction: 10 * time.Minute,
Profile: true,
}
return server.CLIConfig{
EigenDAConfig: eigendaCfg,
RedisCfg: store.RedisConfig{
Endpoint: "127.0.0.1:9001",
Password: "",
DB: 0,
Eviction: 10 * time.Minute,
Profile: true,
},
}
}

Expand All @@ -70,18 +76,18 @@ func createS3Config(eigendaCfg server.Config) server.CLIConfig {
bucketName := "eigenda-proxy-test-" + RandString(10)
createS3Bucket(bucketName)

eigendaCfg.S3Config = s3.Config{
Profiling: true,
Bucket: bucketName,
Path: "",
Endpoint: "localhost:4566",
AccessKeySecret: "minioadmin",
AccessKeyID: "minioadmin",
CredentialType: s3.CredentialTypeStatic,
Backup: false,
}
return server.CLIConfig{
EigenDAConfig: eigendaCfg,
S3Config: store.S3Config{
Profiling: true,
Bucket: bucketName,
Path: "",
Endpoint: "localhost:4566",
AccessKeySecret: "minioadmin",
AccessKeyID: "minioadmin",
S3CredentialType: store.S3CredentialStatic,
Backup: false,
},
}
}

Expand All @@ -105,28 +111,39 @@ func TestSuiteConfig(t *testing.T, testCfg *Cfg) server.CLIConfig {
pollInterval = time.Minute * 1
}

maxBlobLengthBytes, err := utils.ParseBytesAmount("16mib")
require.NoError(t, err)
eigendaCfg := server.Config{
ClientConfig: clients.EigenDAClientConfig{
EdaClientConfig: clients.EigenDAClientConfig{
RPC: holeskyDA,
StatusQueryTimeout: time.Minute * 45,
StatusQueryRetryInterval: pollInterval,
DisableTLS: false,
SignerPrivateKeyHex: pk,
},
EthRPC: ethRPC,
SvcManagerAddr: "0xD4A7E1Bd8015057293f0D0A557088c286942e84b", // incompatible with non holeskly networks
CacheDir: "../resources/SRSTables",
G1Path: "../resources/g1.point",
MaxBlobLength: "16mib",
G2PowerOfTauPath: "../resources/g2.point.powerOf2",
PutBlobEncodingVersion: 0x00,
MemstoreEnabled: testCfg.UseMemory,
MemstoreBlobExpiration: testCfg.Expiration,
EthConfirmationDepth: 0,
VerifierConfig: verify.Config{
VerifyCerts: false,
RPCURL: ethRPC,
SvcManagerAddr: "0xD4A7E1Bd8015057293f0D0A557088c286942e84b", // incompatible with non holeskly networks
EthConfirmationDepth: 0,
KzgConfig: &kzg.KzgConfig{
G1Path: "../resources/g1.point",
G2PowerOf2Path: "../resources/g2.point.powerOf2",
CacheDir: "../resources/SRSTables",
SRSOrder: 268435456,
SRSNumberToLoad: maxBlobLengthBytes / 32,
NumWorker: uint64(runtime.GOMAXPROCS(0)),
},
},
MemstoreEnabled: testCfg.UseMemory,
MemstoreConfig: memstore.Config{
BlobExpiration: testCfg.Expiration,
MaxBlobSizeBytes: maxBlobLengthBytes,
},
}

if testCfg.UseMemory {
eigendaCfg.ClientConfig.SignerPrivateKeyHex = "0000000000000000000100000000000000000000000000000000000000000000"
eigendaCfg.EdaClientConfig.SignerPrivateKeyHex = "0000000000000000000100000000000000000000000000000000000000000000"
}

var cfg server.CLIConfig
Expand Down
121 changes: 121 additions & 0 deletions flags/eigendaflags/cli.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package eigendaflags

import (
"time"

"github.com/Layr-Labs/eigenda/api/clients"
"github.com/Layr-Labs/eigenda/api/clients/codecs"
"github.com/urfave/cli/v2"
)

// TODO: we should eventually move all of these flags into the eigenda repo

var (
DisperserRPCFlagName = withFlagPrefix("disperser-rpc")
StatusQueryRetryIntervalFlagName = withFlagPrefix("status-query-retry-interval")
StatusQueryTimeoutFlagName = withFlagPrefix("status-query-timeout")
DisableTLSFlagName = withFlagPrefix("disable-tls")
ResponseTimeoutFlagName = withFlagPrefix("response-timeout")
CustomQuorumIDsFlagName = withFlagPrefix("custom-quorum-ids")
SignerPrivateKeyHexFlagName = withFlagPrefix("signer-private-key-hex")
PutBlobEncodingVersionFlagName = withFlagPrefix("put-blob-encoding-version")
DisablePointVerificationModeFlagName = withFlagPrefix("disable-point-verification-mode")
WaitForFinalizationFlagName = withFlagPrefix("wait-for-finalization")
)

func withFlagPrefix(s string) string {
return "eigenda." + s
}

func withEnvPrefix(envPrefix, s string) []string {
return []string{envPrefix + "_EIGENDA_" + s}
}

// CLIFlags ... used for EigenDA client configuration
func CLIFlags(envPrefix, category string) []cli.Flag {
return []cli.Flag{
&cli.StringFlag{
Name: DisperserRPCFlagName,
Usage: "RPC endpoint of the EigenDA disperser.",
EnvVars: withEnvPrefix(envPrefix, "DISPERSER_RPC"),
Category: category,
},
&cli.DurationFlag{
Name: StatusQueryTimeoutFlagName,
Usage: "Duration to wait for a blob to finalize after being sent for dispersal. Default is 30 minutes.",
Value: 30 * time.Minute,
EnvVars: withEnvPrefix(envPrefix, "STATUS_QUERY_TIMEOUT"),
Category: category,
},
&cli.DurationFlag{
Name: StatusQueryRetryIntervalFlagName,
Usage: "Interval between retries when awaiting network blob finalization. Default is 5 seconds.",
Value: 5 * time.Second,
EnvVars: withEnvPrefix(envPrefix, "STATUS_QUERY_INTERVAL"),
Category: category,
},
&cli.BoolFlag{
Name: DisableTLSFlagName,
Usage: "Disable TLS for gRPC communication with the EigenDA disperser. Default is false.",
Value: false,
EnvVars: withEnvPrefix(envPrefix, "GRPC_DISABLE_TLS"),
Category: category,
},
&cli.DurationFlag{
Name: ResponseTimeoutFlagName,
Usage: "Total time to wait for a response from the EigenDA disperser. Default is 60 seconds.",
Value: 60 * time.Second,
EnvVars: withEnvPrefix(envPrefix, "RESPONSE_TIMEOUT"),
Category: category,
},
&cli.UintSliceFlag{
Name: CustomQuorumIDsFlagName,
Usage: "Custom quorum IDs for writing blobs. Should not include default quorums 0 or 1.",
Value: cli.NewUintSlice(),
EnvVars: withEnvPrefix(envPrefix, "CUSTOM_QUORUM_IDS"),
Category: category,
},
&cli.StringFlag{
Name: SignerPrivateKeyHexFlagName,
Usage: "Hex-encoded signer private key. This key should not be associated with an Ethereum address holding any funds.",
EnvVars: withEnvPrefix(envPrefix, "SIGNER_PRIVATE_KEY_HEX"),
Category: category,
},
&cli.UintFlag{
Name: PutBlobEncodingVersionFlagName,
Usage: "Blob encoding version to use when writing blobs from the high-level interface.",
EnvVars: withEnvPrefix(envPrefix, "PUT_BLOB_ENCODING_VERSION"),
Value: 0,
Category: category,
},
&cli.BoolFlag{
Name: DisablePointVerificationModeFlagName,
Usage: "Disable point verification mode. This mode performs IFFT on data before writing and FFT on data after reading. Disabling requires supplying the entire blob for verification against the KZG commitment.",
EnvVars: withEnvPrefix(envPrefix, "DISABLE_POINT_VERIFICATION_MODE"),
Value: false,
Category: category,
},
&cli.BoolFlag{
Name: WaitForFinalizationFlagName,
Usage: "Wait for blob finalization before returning from PutBlob.",
EnvVars: withEnvPrefix(envPrefix, "WAIT_FOR_FINALIZATION"),
Value: false,
Category: category,
},
}
}

func ReadConfig(ctx *cli.Context) clients.EigenDAClientConfig {
return clients.EigenDAClientConfig{
RPC: ctx.String(DisperserRPCFlagName),
StatusQueryRetryInterval: ctx.Duration(StatusQueryRetryIntervalFlagName),
StatusQueryTimeout: ctx.Duration(StatusQueryTimeoutFlagName),
DisableTLS: ctx.Bool(DisableTLSFlagName),
ResponseTimeout: ctx.Duration(ResponseTimeoutFlagName),
CustomQuorumIDs: ctx.UintSlice(CustomQuorumIDsFlagName),
SignerPrivateKeyHex: ctx.String(SignerPrivateKeyHexFlagName),
PutBlobEncodingVersion: codecs.BlobEncodingVersion(ctx.Uint(PutBlobEncodingVersionFlagName)),
DisablePointVerificationMode: ctx.Bool(DisablePointVerificationModeFlagName),
WaitForFinalization: ctx.Bool(WaitForFinalizationFlagName),
}
}
Loading

0 comments on commit 2600bad

Please sign in to comment.