From e57024c48c8a0de5660688c260e1ca357d4e21e1 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Thu, 29 Dec 2022 16:21:18 -0700 Subject: [PATCH 01/22] init commit optimistic relay --- datastore/redis.go | 30 ++++++---- datastore/utils.go | 8 ++- services/api/service.go | 92 +++++++++++++++++++---------- services/housekeeper/housekeeper.go | 5 +- 4 files changed, 89 insertions(+), 46 deletions(-) diff --git a/datastore/redis.go b/datastore/redis.go index 1e15536f..5ab54dae 100644 --- a/datastore/redis.go +++ b/datastore/redis.go @@ -32,14 +32,22 @@ var ( ErrFailedUpdatingTopBidNoBids = errors.New("failed to update top bid because no bids were found") ) -type BlockBuilderStatus string +type BlockBuilderStatusStr string var ( - RedisBlockBuilderStatusLowPrio BlockBuilderStatus = "" - RedisBlockBuilderStatusHighPrio BlockBuilderStatus = "high-prio" - RedisBlockBuilderStatusBlacklisted BlockBuilderStatus = "blacklisted" + RedisBlockBuilderStatusLowPrio BlockBuilderStatusStr = "" + RedisBlockBuilderStatusHighPrio BlockBuilderStatusStr = "high-prio" + RedisBlockBuilderStatusBlacklisted BlockBuilderStatusStr = "blacklisted" + RedisBlockBuilderStatusOptimistic BlockBuilderStatusStr = "optimistic" ) +type BlockBuilderStatus struct { + LowPrio bool + HighPrio bool + Blacklisted bool + Optimistic bool +} + func PubkeyHexToLowerStr(pk types.PubkeyHex) string { return strings.ToLower(string(pk)) } @@ -323,18 +331,20 @@ func (r *RedisCache) GetBidTrace(slot uint64, proposerPubkey, blockHash string) return resp, err } -func (r *RedisCache) SetBlockBuilderStatus(builderPubkey string, status BlockBuilderStatus) (err error) { +func (r *RedisCache) SetBlockBuilderStatus(builderPubkey string, status BlockBuilderStatusStr) (err error) { return r.client.HSet(context.Background(), r.keyBlockBuilderStatus, builderPubkey, string(status)).Err() } -func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (isHighPrio, isBlacklisted bool, err error) { +func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (status BlockBuilderStatus, err error) { res, err := r.client.HGet(context.Background(), r.keyBlockBuilderStatus, builderPubkey).Result() if errors.Is(err, redis.Nil) { - return false, false, nil + return status, nil } - isHighPrio = BlockBuilderStatus(res) == RedisBlockBuilderStatusHighPrio - isBlacklisted = BlockBuilderStatus(res) == RedisBlockBuilderStatusBlacklisted - return isHighPrio, isBlacklisted, err + status.HighPrio = BlockBuilderStatusStr(res) == RedisBlockBuilderStatusHighPrio + status.LowPrio = !status.HighPrio + status.Blacklisted = BlockBuilderStatusStr(res) == RedisBlockBuilderStatusBlacklisted + status.Optimistic = BlockBuilderStatusStr(res) == RedisBlockBuilderStatusOptimistic + return status, err } func (r *RedisCache) GetBuilderLatestPayloadReceivedAt(slot uint64, builderPubkey, parentHash, proposerPubkey string) (int64, error) { diff --git a/datastore/utils.go b/datastore/utils.go index 6a9c8f96..83537c61 100644 --- a/datastore/utils.go +++ b/datastore/utils.go @@ -1,9 +1,11 @@ package datastore -func MakeBlockBuilderStatus(isHighPrio, isBlacklisted bool) BlockBuilderStatus { - if isBlacklisted { +func MakeBlockBuilderStatus(s BlockBuilderStatus) BlockBuilderStatusStr { + if s.Blacklisted { return RedisBlockBuilderStatusBlacklisted - } else if isHighPrio { + } else if s.Optimistic { + return RedisBlockBuilderStatusOptimistic + } else if s.HighPrio { return RedisBlockBuilderStatusHighPrio } else { return RedisBlockBuilderStatusLowPrio diff --git a/services/api/service.go b/services/api/service.go index 439ad6c0..b8eae8d7 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -839,7 +839,14 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) return } signedBeaconBlock := SignedBlindedBeaconBlockToBeaconBlock(payload, getPayloadResp.Data) - _, _ = api.beaconClient.PublishBlock(signedBeaconBlock) // errors are logged inside + _, _ = api.beaconClient.PublishBlock(signedBeaconBlock) // errors are logged inside. + // Analyze published blocks for invalid blocks aysnchronously. + // Plan: + // 1. Write blocks to an OptimisticPublishedBlocks channel. + // 2. A separate thread will read from the channel and distribute work + // to the validating nodes. + // 3. If any published block is determined to be invalid, we update + // the builder status to low-prio and calculate reimbursement. }() } @@ -945,10 +952,11 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } } - builderIsHighPrio, builderIsBlacklisted, err := api.redis.GetBlockBuilderStatus(payload.Message.BuilderPubkey.String()) + builderStatus, err := api.redis.GetBlockBuilderStatus(payload.Message.BuilderPubkey.String()) log = log.WithFields(logrus.Fields{ - "builderIsHighPrio": builderIsHighPrio, - "builderIsBlacklisted": builderIsBlacklisted, + "builderIsHighPrio": builderStatus.HighPrio, + "builderIsBlacklisted": builderStatus.Blacklisted, + "builderIsOptimistic": builderStatus.Optimistic, }) if err != nil { log.WithError(err).Error("could not get block builder status") @@ -985,7 +993,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque return } - if builderIsBlacklisted { + if builderStatus.Blacklisted { log.Info("builder is blacklisted") time.Sleep(200 * time.Millisecond) w.WriteHeader(http.StatusOK) @@ -993,7 +1001,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } // In case only high-prio requests are accepted, fail others - if api.ffDisableLowPrioBuilders && !builderIsHighPrio { + if api.ffDisableLowPrioBuilders && !builderStatus.HighPrio { log.Info("rejecting low-prio builder (ff-disable-low-prio-builders)") time.Sleep(200 * time.Millisecond) w.WriteHeader(http.StatusOK) @@ -1001,11 +1009,12 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } log = log.WithFields(logrus.Fields{ - "builderHighPrio": builderIsHighPrio, - "proposerPubkey": payload.Message.ProposerPubkey.String(), - "parentHash": payload.Message.ParentHash.String(), - "value": payload.Message.Value.String(), - "tx": len(payload.ExecutionPayload.Transactions), + "builderHighPrio": builderStatus.HighPrio, + "builderOptimistic": builderStatus.Optimistic, + "proposerPubkey": payload.Message.ProposerPubkey.String(), + "parentHash": payload.Message.ParentHash.String(), + "value": payload.Message.Value.String(), + "tx": len(payload.ExecutionPayload.Transactions), }) if payload.Message.Slot <= api.headSlot.Load() { @@ -1068,27 +1077,38 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } }() - // Simulate the block submission and save to db - t := time.Now() - validationRequestPayload := &BuilderBlockValidationRequest{ - BuilderSubmitBlockRequest: *payload, - RegisteredGasLimit: slotDuty.GasLimit, - } - simErr = api.blockSimRateLimiter.send(req.Context(), validationRequestPayload, builderIsHighPrio) - - if simErr != nil { - log = log.WithField("simErr", simErr.Error()) - log.WithError(simErr).WithFields(logrus.Fields{ - "duration": time.Since(t).Seconds(), - "numWaiting": api.blockSimRateLimiter.currentCounter(), - }).Info("block validation failed") - api.RespondError(w, http.StatusBadRequest, simErr.Error()) - return + // Only perform simulation on hot path if we are in non-optimistic mode. + if !builderStatus.Optimistic { + // Simulate the block submission and save to db. + t := time.Now() + validationRequestPayload := &BuilderBlockValidationRequest{ + BuilderSubmitBlockRequest: *payload, + RegisteredGasLimit: slotDuty.GasLimit, + } + simErr = api.blockSimRateLimiter.send(req.Context(), validationRequestPayload, builderStatus.HighPrio) + + if simErr != nil { + log = log.WithField("simErr", simErr.Error()) + log.WithError(simErr).WithFields(logrus.Fields{ + "duration": time.Since(t).Seconds(), + "numWaiting": api.blockSimRateLimiter.currentCounter(), + }).Info("block validation failed") + api.RespondError(w, http.StatusBadRequest, simErr.Error()) + return + } else { + log.WithFields(logrus.Fields{ + "duration": time.Since(t).Seconds(), + "numWaiting": api.blockSimRateLimiter.currentCounter(), + }).Info("block validation successful") + } } else { - log.WithFields(logrus.Fields{ - "duration": time.Since(t).Seconds(), - "numWaiting": api.blockSimRateLimiter.currentCounter(), - }).Info("block validation successful") + // In optmistic mode, simulate the block asynchronously. + // Plan: + // 1. Write blocks to an Optimistic Blocks channel. + // 2. A separate thread will read from the channel and distribute work + // to the validating nodes. + // 3. If any block is determined to be invald, we update the builder + // status to low-prio. } // Ensure this request is still the latest one @@ -1200,13 +1220,21 @@ func (api *RelayAPI) handleInternalBuilderStatus(w http.ResponseWriter, req *htt args := req.URL.Query() isHighPrio := args.Get("high_prio") == "true" isBlacklisted := args.Get("blacklisted") == "true" + isOptimistic := args.Get("optimistic") == "true" api.log.WithFields(logrus.Fields{ "builderPubkey": builderPubkey, "isHighPrio": isHighPrio, "isBlacklisted": isBlacklisted, + "isOptimistic": isOptimistic, }).Info("updating builder status") + status := datastore.BlockBuilderStatus{ + HighPrio: isHighPrio, + LowPrio: !isHighPrio, + Blacklisted: isBlacklisted, + Optimistic: isOptimistic, + } - newStatus := datastore.MakeBlockBuilderStatus(isHighPrio, isBlacklisted) + newStatus := datastore.MakeBlockBuilderStatus(status) err := api.redis.SetBlockBuilderStatus(builderPubkey, newStatus) if err != nil { api.log.WithError(err).Error("could not set block builder status in redis") diff --git a/services/housekeeper/housekeeper.go b/services/housekeeper/housekeeper.go index e87bc3cd..b9ebf340 100644 --- a/services/housekeeper/housekeeper.go +++ b/services/housekeeper/housekeeper.go @@ -338,7 +338,10 @@ func (hk *Housekeeper) updateBlockBuildersInRedis() { hk.log.Infof("updating %d block builders in Redis...", len(builders)) for _, builder := range builders { - status := datastore.MakeBlockBuilderStatus(builder.IsHighPrio, builder.IsBlacklisted) + status := datastore.MakeBlockBuilderStatus(datastore.BlockBuilderStatus{ + HighPrio: builder.IsHighPrio, + Blacklisted: builder.IsBlacklisted, + }) hk.log.Infof("updating block builder in Redis: %s - %s", builder.BuilderPubkey, status) err = hk.redis.SetBlockBuilderStatus(builder.BuilderPubkey, status) if err != nil { From 95e937a18bdeed667db8f5aa07bf2d7acb21a846 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Fri, 30 Dec 2022 23:14:27 -0700 Subject: [PATCH 02/22] optimistic relay updates --- database/types.go | 1 + datastore/redis.go | 56 ++++++++---- datastore/utils.go | 15 ++-- services/api/service.go | 131 ++++++++++++++++++---------- services/housekeeper/housekeeper.go | 13 ++- 5 files changed, 141 insertions(+), 75 deletions(-) diff --git a/database/types.go b/database/types.go index 7ebe8e7e..e1c7f629 100644 --- a/database/types.go +++ b/database/types.go @@ -188,6 +188,7 @@ type BlockBuilderEntry struct { BuilderPubkey string `db:"builder_pubkey" json:"builder_pubkey"` Description string `db:"description" json:"description"` + IsOptimistic bool `db:"is_optimistic" json:"is_optimistic"` IsHighPrio bool `db:"is_high_prio" json:"is_high_prio"` IsBlacklisted bool `db:"is_blacklisted" json:"is_blacklisted"` diff --git a/datastore/redis.go b/datastore/redis.go index 5ab54dae..582a03c2 100644 --- a/datastore/redis.go +++ b/datastore/redis.go @@ -32,20 +32,35 @@ var ( ErrFailedUpdatingTopBidNoBids = errors.New("failed to update top bid because no bids were found") ) -type BlockBuilderStatusStr string +type BlockBuilderStatus string var ( - RedisBlockBuilderStatusLowPrio BlockBuilderStatusStr = "" - RedisBlockBuilderStatusHighPrio BlockBuilderStatusStr = "high-prio" - RedisBlockBuilderStatusBlacklisted BlockBuilderStatusStr = "blacklisted" - RedisBlockBuilderStatusOptimistic BlockBuilderStatusStr = "optimistic" + RedisBlockBuilderStatusLowPrio BlockBuilderStatus = "" + RedisBlockBuilderStatusHighPrio BlockBuilderStatus = "high-prio" + RedisBlockBuilderStatusOptimistic BlockBuilderStatus = "optimistic" + RedisBlockBuilderStatusBlacklisted BlockBuilderStatus = "blacklisted" ) -type BlockBuilderStatus struct { - LowPrio bool - HighPrio bool - Blacklisted bool - Optimistic bool +type BlockBuilderStatusCode uint64 + +const ( + LowPrio BlockBuilderStatusCode = iota + HighPrio + Optimistic + Blacklisted +) + +func (b BlockBuilderStatusCode) String() string { + switch b { + case Optimistic: + return string(RedisBlockBuilderStatusOptimistic) + case HighPrio: + return string(RedisBlockBuilderStatusHighPrio) + case Blacklisted: + return string(RedisBlockBuilderStatusBlacklisted) + default: + return string(RedisBlockBuilderStatusLowPrio) + } } func PubkeyHexToLowerStr(pk types.PubkeyHex) string { @@ -331,20 +346,25 @@ func (r *RedisCache) GetBidTrace(slot uint64, proposerPubkey, blockHash string) return resp, err } -func (r *RedisCache) SetBlockBuilderStatus(builderPubkey string, status BlockBuilderStatusStr) (err error) { +func (r *RedisCache) SetBlockBuilderStatus(builderPubkey string, status BlockBuilderStatus) (err error) { return r.client.HSet(context.Background(), r.keyBlockBuilderStatus, builderPubkey, string(status)).Err() } -func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (status BlockBuilderStatus, err error) { +func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (code BlockBuilderStatusCode, err error) { res, err := r.client.HGet(context.Background(), r.keyBlockBuilderStatus, builderPubkey).Result() if errors.Is(err, redis.Nil) { - return status, nil + return code, nil + } + switch status := BlockBuilderStatus(res); status { + case RedisBlockBuilderStatusHighPrio: + return HighPrio, nil + case RedisBlockBuilderStatusOptimistic: + return Optimistic, nil + case RedisBlockBuilderStatusBlacklisted: + return Blacklisted, nil + default: + return LowPrio, nil } - status.HighPrio = BlockBuilderStatusStr(res) == RedisBlockBuilderStatusHighPrio - status.LowPrio = !status.HighPrio - status.Blacklisted = BlockBuilderStatusStr(res) == RedisBlockBuilderStatusBlacklisted - status.Optimistic = BlockBuilderStatusStr(res) == RedisBlockBuilderStatusOptimistic - return status, err } func (r *RedisCache) GetBuilderLatestPayloadReceivedAt(slot uint64, builderPubkey, parentHash, proposerPubkey string) (int64, error) { diff --git a/datastore/utils.go b/datastore/utils.go index 83537c61..d8f819f9 100644 --- a/datastore/utils.go +++ b/datastore/utils.go @@ -1,13 +1,14 @@ package datastore -func MakeBlockBuilderStatus(s BlockBuilderStatus) BlockBuilderStatusStr { - if s.Blacklisted { - return RedisBlockBuilderStatusBlacklisted - } else if s.Optimistic { - return RedisBlockBuilderStatusOptimistic - } else if s.HighPrio { +func MakeBlockBuilderStatus(c BlockBuilderStatusCode) BlockBuilderStatus { + switch c { + case HighPrio: return RedisBlockBuilderStatusHighPrio - } else { + case Optimistic: + return RedisBlockBuilderStatusOptimistic + case Blacklisted: + return RedisBlockBuilderStatusBlacklisted + default: return RedisBlockBuilderStatusLowPrio } } diff --git a/services/api/service.go b/services/api/service.go index b8eae8d7..d59af492 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -98,6 +98,14 @@ type randaoHelper struct { prevRandao string } +// Contains necessary elements to issue a block validation request. +type blockSimOptions struct { + ctx context.Context + highPrio bool + log *logrus.Entry + req *BuilderBlockValidationRequest +} + // RelayAPI represents a single Relay instance type RelayAPI struct { opts RelayAPIOpts @@ -139,6 +147,8 @@ type RelayAPI struct { expectedPrevRandao randaoHelper expectedPrevRandaoLock sync.RWMutex expectedPrevRandaoUpdating uint64 + + optimisticBlockC chan blockSimOptions } // NewRelayAPI creates a new service. if builders is nil, allow any builder @@ -304,6 +314,8 @@ func (api *RelayAPI) StartServer() (err error) { for i := 0; i < numValidatorRegProcessors; i++ { go api.startValidatorRegistrationDBProcessor() } + + go api.startOptimisticBlockProcessor() } // Process current slot @@ -381,6 +393,45 @@ func (api *RelayAPI) startValidatorRegistrationDBProcessor() { } } +func (api *RelayAPI) simulateBlock(opts blockSimOptions) error { + t := time.Now() + simErr := api.blockSimRateLimiter.send(opts.ctx, opts.req, opts.highPrio) + if simErr != nil { + log := opts.log.WithField("simErr", simErr.Error()) + log.WithError(simErr).WithFields(logrus.Fields{ + "duration": time.Since(t).Seconds(), + "numWaiting": api.blockSimRateLimiter.currentCounter(), + }).Info("block validation failed") + } else { + opts.log.WithFields(logrus.Fields{ + "duration": time.Since(t).Seconds(), + "numWaiting": api.blockSimRateLimiter.currentCounter(), + }).Info("block validation successful") + } + return simErr +} + +// startOptimisticBlockProcessor keeps listening on the channel and validating incoming blocks asynchronously +func (api *RelayAPI) startOptimisticBlockProcessor() { + for opts := range api.optimisticBlockC { + err := api.simulateBlock(opts) + if err != nil { + // Validation failed, mark the status of the builder as lowPrio (pessemistic). + newStatus := datastore.MakeBlockBuilderStatus(datastore.LowPrio) + builderPubKey := opts.req.Message.BuilderPubkey.String() + err := api.redis.SetBlockBuilderStatus(builderPubKey, newStatus) + if err != nil { + api.log.WithError(err).Error("could not set block builder status in redis") + } + + err = api.db.SetBlockBuilderStatus(builderPubKey, false /*isHighPrio*/, false /*isBlacklisted*/) + if err != nil { + api.log.WithError(err).Error("could not set block builder status in database") + } + } + } +} + func (api *RelayAPI) processNewSlot(headSlot uint64) { _apiHeadSlot := api.headSlot.Load() if headSlot <= _apiHeadSlot { @@ -952,11 +1003,10 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } } - builderStatus, err := api.redis.GetBlockBuilderStatus(payload.Message.BuilderPubkey.String()) + builderStatusCode, err := api.redis.GetBlockBuilderStatus(payload.Message.BuilderPubkey.String()) log = log.WithFields(logrus.Fields{ - "builderIsHighPrio": builderStatus.HighPrio, - "builderIsBlacklisted": builderStatus.Blacklisted, - "builderIsOptimistic": builderStatus.Optimistic, + "builderStatus": builderStatusCode, + "builderStatusStr": builderStatusCode.String(), }) if err != nil { log.WithError(err).Error("could not get block builder status") @@ -993,7 +1043,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque return } - if builderStatus.Blacklisted { + if builderStatusCode == datastore.Blacklisted { log.Info("builder is blacklisted") time.Sleep(200 * time.Millisecond) w.WriteHeader(http.StatusOK) @@ -1001,7 +1051,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } // In case only high-prio requests are accepted, fail others - if api.ffDisableLowPrioBuilders && !builderStatus.HighPrio { + if api.ffDisableLowPrioBuilders && builderStatusCode != datastore.HighPrio { log.Info("rejecting low-prio builder (ff-disable-low-prio-builders)") time.Sleep(200 * time.Millisecond) w.WriteHeader(http.StatusOK) @@ -1009,12 +1059,12 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } log = log.WithFields(logrus.Fields{ - "builderHighPrio": builderStatus.HighPrio, - "builderOptimistic": builderStatus.Optimistic, - "proposerPubkey": payload.Message.ProposerPubkey.String(), - "parentHash": payload.Message.ParentHash.String(), - "value": payload.Message.Value.String(), - "tx": len(payload.ExecutionPayload.Transactions), + "builderStatus": builderStatusCode, + "builderStatusStr": builderStatusCode.String(), + "proposerPubkey": payload.Message.ProposerPubkey.String(), + "parentHash": payload.Message.ParentHash.String(), + "value": payload.Message.Value.String(), + "tx": len(payload.ExecutionPayload.Transactions), }) if payload.Message.Slot <= api.headSlot.Load() { @@ -1077,38 +1127,26 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } }() - // Only perform simulation on hot path if we are in non-optimistic mode. - if !builderStatus.Optimistic { - // Simulate the block submission and save to db. - t := time.Now() - validationRequestPayload := &BuilderBlockValidationRequest{ + // Construct simulation request. + opts := blockSimOptions{ + ctx: req.Context(), + highPrio: builderStatusCode == datastore.HighPrio, + log: log, + req: &BuilderBlockValidationRequest{ BuilderSubmitBlockRequest: *payload, RegisteredGasLimit: slotDuty.GasLimit, - } - simErr = api.blockSimRateLimiter.send(req.Context(), validationRequestPayload, builderStatus.HighPrio) + }, + } + // Only perform simulation on hot path if we are in non-optimistic mode. + if builderStatusCode != datastore.Optimistic { + simErr = api.simulateBlock(opts) if simErr != nil { - log = log.WithField("simErr", simErr.Error()) - log.WithError(simErr).WithFields(logrus.Fields{ - "duration": time.Since(t).Seconds(), - "numWaiting": api.blockSimRateLimiter.currentCounter(), - }).Info("block validation failed") api.RespondError(w, http.StatusBadRequest, simErr.Error()) - return - } else { - log.WithFields(logrus.Fields{ - "duration": time.Since(t).Seconds(), - "numWaiting": api.blockSimRateLimiter.currentCounter(), - }).Info("block validation successful") } } else { - // In optmistic mode, simulate the block asynchronously. - // Plan: - // 1. Write blocks to an Optimistic Blocks channel. - // 2. A separate thread will read from the channel and distribute work - // to the validating nodes. - // 3. If any block is determined to be invald, we update the builder - // status to low-prio. + // Write optimistic block to channel for async validation. + api.optimisticBlockC <- opts } // Ensure this request is still the latest one @@ -1218,23 +1256,24 @@ func (api *RelayAPI) handleInternalBuilderStatus(w http.ResponseWriter, req *htt return } else if req.Method == http.MethodPost || req.Method == http.MethodPut || req.Method == http.MethodPatch { args := req.URL.Query() + isOptimistic := args.Get("optimistic") == "true" isHighPrio := args.Get("high_prio") == "true" isBlacklisted := args.Get("blacklisted") == "true" - isOptimistic := args.Get("optimistic") == "true" api.log.WithFields(logrus.Fields{ "builderPubkey": builderPubkey, + "isOptimistic": isOptimistic, "isHighPrio": isHighPrio, "isBlacklisted": isBlacklisted, - "isOptimistic": isOptimistic, }).Info("updating builder status") - status := datastore.BlockBuilderStatus{ - HighPrio: isHighPrio, - LowPrio: !isHighPrio, - Blacklisted: isBlacklisted, - Optimistic: isOptimistic, + code := datastore.LowPrio + if isBlacklisted { + code = datastore.Blacklisted + } else if isOptimistic { + code = datastore.Optimistic + } else if isHighPrio { + code = datastore.HighPrio } - - newStatus := datastore.MakeBlockBuilderStatus(status) + newStatus := datastore.MakeBlockBuilderStatus(code) err := api.redis.SetBlockBuilderStatus(builderPubkey, newStatus) if err != nil { api.log.WithError(err).Error("could not set block builder status in redis") diff --git a/services/housekeeper/housekeeper.go b/services/housekeeper/housekeeper.go index b9ebf340..6c58ef40 100644 --- a/services/housekeeper/housekeeper.go +++ b/services/housekeeper/housekeeper.go @@ -338,10 +338,15 @@ func (hk *Housekeeper) updateBlockBuildersInRedis() { hk.log.Infof("updating %d block builders in Redis...", len(builders)) for _, builder := range builders { - status := datastore.MakeBlockBuilderStatus(datastore.BlockBuilderStatus{ - HighPrio: builder.IsHighPrio, - Blacklisted: builder.IsBlacklisted, - }) + code := datastore.LowPrio + if builder.IsOptimistic { + code = datastore.Optimistic + } else if builder.IsHighPrio { + code = datastore.HighPrio + } else if builder.IsBlacklisted { + code = datastore.Blacklisted + } + status := datastore.MakeBlockBuilderStatus(code) hk.log.Infof("updating block builder in Redis: %s - %s", builder.BuilderPubkey, status) err = hk.redis.SetBlockBuilderStatus(builder.BuilderPubkey, status) if err != nil { From e666a4b5e00d3d59d7522da0100bda8477239034 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Sat, 31 Dec 2022 09:08:15 -0700 Subject: [PATCH 03/22] db updates --- common/common.go | 9 +++ database/database.go | 16 ++--- database/mockdb.go | 2 +- database/types.go | 4 +- datastore/redis.go | 27 +++----- datastore/utils.go | 10 +-- services/api/service.go | 95 ++++++++++++++++++++++------- services/housekeeper/housekeeper.go | 9 +-- 8 files changed, 107 insertions(+), 65 deletions(-) diff --git a/common/common.go b/common/common.go index 22f92c86..01966253 100644 --- a/common/common.go +++ b/common/common.go @@ -21,3 +21,12 @@ type HTTPServerTimeouts struct { Write time.Duration // Timeout for writes. None if 0. Idle time.Duration // Timeout to disconnect idle client connections. None if 0. } + +type BlockBuilderStatusCode uint8 + +const ( + LowPrio BlockBuilderStatusCode = iota + HighPrio + Optimistic + Blacklisted +) diff --git a/database/database.go b/database/database.go index 1566dc8b..9df54f55 100644 --- a/database/database.go +++ b/database/database.go @@ -41,7 +41,7 @@ type IDatabaseService interface { GetBlockBuilders() ([]*BlockBuilderEntry, error) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderEntry, error) - SetBlockBuilderStatus(pubkey string, isHighPrio, isBlacklisted bool) error + SetBlockBuilderStatus(pubkey string, status common.BlockBuilderStatusCode) error UpsertBlockBuilderEntryAfterSubmission(lastSubmission *BuilderBlockSubmissionEntry, isError bool) error IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error } @@ -436,8 +436,8 @@ func (s *DatabaseService) UpsertBlockBuilderEntryAfterSubmission(lastSubmission // Upsert query := `INSERT INTO ` + vars.TableBlockBuilder + ` - (builder_pubkey, description, is_high_prio, is_blacklisted, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror) VALUES - (:builder_pubkey, :description, :is_high_prio, :is_blacklisted, :last_submission_id, :last_submission_slot, :num_submissions_total, :num_submissions_simerror) + (builder_pubkey, description, builder_status, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror) VALUES + (:builder_pubkey, :description, :builder_status, :last_submission_id, :last_submission_slot, :num_submissions_total, :num_submissions_simerror) ON CONFLICT (builder_pubkey) DO UPDATE SET last_submission_id = :last_submission_id, last_submission_slot = :last_submission_slot, @@ -448,22 +448,22 @@ func (s *DatabaseService) UpsertBlockBuilderEntryAfterSubmission(lastSubmission } func (s *DatabaseService) GetBlockBuilders() ([]*BlockBuilderEntry, error) { - query := `SELECT id, inserted_at, builder_pubkey, description, is_high_prio, is_blacklisted, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` ORDER BY id ASC;` + query := `SELECT id, inserted_at, builder_pubkey, description, builder_status, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` ORDER BY id ASC;` entries := []*BlockBuilderEntry{} err := s.DB.Select(&entries, query) return entries, err } func (s *DatabaseService) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderEntry, error) { - query := `SELECT id, inserted_at, builder_pubkey, description, is_high_prio, is_blacklisted, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` WHERE builder_pubkey=$1;` + query := `SELECT id, inserted_at, builder_pubkey, description, builder_status, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` WHERE builder_pubkey=$1;` entry := &BlockBuilderEntry{} err := s.DB.Get(entry, query, pubkey) return entry, err } -func (s *DatabaseService) SetBlockBuilderStatus(pubkey string, isHighPrio, isBlacklisted bool) error { - query := `UPDATE ` + vars.TableBlockBuilder + ` SET is_high_prio=$1, is_blacklisted=$2 WHERE builder_pubkey=$3;` - _, err := s.DB.Exec(query, isHighPrio, isBlacklisted, pubkey) +func (s *DatabaseService) SetBlockBuilderStatus(pubkey string, builderStatus common.BlockBuilderStatusCode) error { + query := `UPDATE ` + vars.TableBlockBuilder + ` SET builder_status=$1 WHERE builder_pubkey=$3;` + _, err := s.DB.Exec(query, uint8(builderStatus), pubkey) return err } diff --git a/database/mockdb.go b/database/mockdb.go index e6bc405e..4fb10fdb 100644 --- a/database/mockdb.go +++ b/database/mockdb.go @@ -89,7 +89,7 @@ func (db MockDB) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderEntry, err return nil, nil } -func (db MockDB) SetBlockBuilderStatus(pubkey string, isHighPrio, isBlacklisted bool) error { +func (db MockDB) SetBlockBuilderStatus(pubkey string, builderStatus common.BlockBuilderStatusCode) error { return nil } diff --git a/database/types.go b/database/types.go index e1c7f629..2828b4fc 100644 --- a/database/types.go +++ b/database/types.go @@ -188,9 +188,7 @@ type BlockBuilderEntry struct { BuilderPubkey string `db:"builder_pubkey" json:"builder_pubkey"` Description string `db:"description" json:"description"` - IsOptimistic bool `db:"is_optimistic" json:"is_optimistic"` - IsHighPrio bool `db:"is_high_prio" json:"is_high_prio"` - IsBlacklisted bool `db:"is_blacklisted" json:"is_blacklisted"` + BuilderStatus uint8 `db:"builder_status" json:"builder_status"` LastSubmissionID sql.NullInt64 `db:"last_submission_id" json:"last_submission_id"` LastSubmissionSlot uint64 `db:"last_submission_slot" json:"last_submission_slot"` diff --git a/datastore/redis.go b/datastore/redis.go index 582a03c2..73daa6f2 100644 --- a/datastore/redis.go +++ b/datastore/redis.go @@ -41,22 +41,13 @@ var ( RedisBlockBuilderStatusBlacklisted BlockBuilderStatus = "blacklisted" ) -type BlockBuilderStatusCode uint64 - -const ( - LowPrio BlockBuilderStatusCode = iota - HighPrio - Optimistic - Blacklisted -) - -func (b BlockBuilderStatusCode) String() string { +func StatusStringFromCode(b common.BlockBuilderStatusCode) string { switch b { - case Optimistic: + case common.Optimistic: return string(RedisBlockBuilderStatusOptimistic) - case HighPrio: + case common.HighPrio: return string(RedisBlockBuilderStatusHighPrio) - case Blacklisted: + case common.Blacklisted: return string(RedisBlockBuilderStatusBlacklisted) default: return string(RedisBlockBuilderStatusLowPrio) @@ -350,20 +341,20 @@ func (r *RedisCache) SetBlockBuilderStatus(builderPubkey string, status BlockBui return r.client.HSet(context.Background(), r.keyBlockBuilderStatus, builderPubkey, string(status)).Err() } -func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (code BlockBuilderStatusCode, err error) { +func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (code common.BlockBuilderStatusCode, err error) { res, err := r.client.HGet(context.Background(), r.keyBlockBuilderStatus, builderPubkey).Result() if errors.Is(err, redis.Nil) { return code, nil } switch status := BlockBuilderStatus(res); status { case RedisBlockBuilderStatusHighPrio: - return HighPrio, nil + return common.HighPrio, nil case RedisBlockBuilderStatusOptimistic: - return Optimistic, nil + return common.Optimistic, nil case RedisBlockBuilderStatusBlacklisted: - return Blacklisted, nil + return common.Blacklisted, nil default: - return LowPrio, nil + return common.LowPrio, nil } } diff --git a/datastore/utils.go b/datastore/utils.go index d8f819f9..223e4d11 100644 --- a/datastore/utils.go +++ b/datastore/utils.go @@ -1,12 +1,14 @@ package datastore -func MakeBlockBuilderStatus(c BlockBuilderStatusCode) BlockBuilderStatus { +import "github.com/flashbots/mev-boost-relay/common" + +func MakeBlockBuilderStatus(c common.BlockBuilderStatusCode) BlockBuilderStatus { switch c { - case HighPrio: + case common.HighPrio: return RedisBlockBuilderStatusHighPrio - case Optimistic: + case common.Optimistic: return RedisBlockBuilderStatusOptimistic - case Blacklisted: + case common.Blacklisted: return RedisBlockBuilderStatusBlacklisted default: return RedisBlockBuilderStatusLowPrio diff --git a/services/api/service.go b/services/api/service.go index d59af492..04e8ed67 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -98,7 +98,7 @@ type randaoHelper struct { prevRandao string } -// Contains necessary elements to issue a block validation request. +// Data needed to issue a block validation request. type blockSimOptions struct { ctx context.Context highPrio bool @@ -106,6 +106,15 @@ type blockSimOptions struct { req *BuilderBlockValidationRequest } +// Data needed to initiate a proposer refund. +type proposerRefundOptions struct { + proposerPK string + builderPK string + amount types.U256Str + slot uint64 + bidTrace types.BidTrace +} + // RelayAPI represents a single Relay instance type RelayAPI struct { opts RelayAPIOpts @@ -148,7 +157,9 @@ type RelayAPI struct { expectedPrevRandaoLock sync.RWMutex expectedPrevRandaoUpdating uint64 + // Channel used to process optimistic blocks asynchronously. optimisticBlockC chan blockSimOptions + proposerRefundC chan proposerRefundOptions } // NewRelayAPI creates a new service. if builders is nil, allow any builder @@ -207,6 +218,8 @@ func NewRelayAPI(opts RelayAPIOpts) (api *RelayAPI, err error) { activeValidatorC: make(chan types.PubkeyHex, 450_000), validatorRegC: make(chan types.SignedValidatorRegistration, 450_000), + optimisticBlockC: make(chan blockSimOptions, 450_000), + proposerRefundC: make(chan proposerRefundOptions, 450_000), } if os.Getenv("FORCE_GET_HEADER_204") == "1" { @@ -296,6 +309,9 @@ func (api *RelayAPI) StartServer() (err error) { if api.opts.BlockBuilderAPI { // Get current proposer duties blocking before starting, to have them ready api.updateProposerDuties(bestSyncStatus.HeadSlot) + + // TODO(mikeneuder): consider if we should use >1 optimisticBlockProcessors. + go api.startOptimisticBlockProcessor() } // start things specific for the proposer API @@ -315,7 +331,8 @@ func (api *RelayAPI) StartServer() (err error) { go api.startValidatorRegistrationDBProcessor() } - go api.startOptimisticBlockProcessor() + // TODO(mikeneuder): consider if we should use >1 proposerRefundProcessors. + go api.startProposerRefundProcessor() } // Process current slot @@ -328,6 +345,8 @@ func (api *RelayAPI) StartServer() (err error) { for { headEvent := <-c api.processNewSlot(headEvent.Slot) + // TODO(mikeneuder): Add checks to make sure we haven't + // optimistically built on top of a non-simulated slot. } }() @@ -393,6 +412,7 @@ func (api *RelayAPI) startValidatorRegistrationDBProcessor() { } } +// simulateBlock sends a request for a block simulation to blockSimRateLimiter. func (api *RelayAPI) simulateBlock(opts blockSimOptions) error { t := time.Now() simErr := api.blockSimRateLimiter.send(opts.ctx, opts.req, opts.highPrio) @@ -411,20 +431,20 @@ func (api *RelayAPI) simulateBlock(opts blockSimOptions) error { return simErr } -// startOptimisticBlockProcessor keeps listening on the channel and validating incoming blocks asynchronously +// startOptimisticBlockProcessor keeps listening on the channel and validating incoming blocks asynchronously. func (api *RelayAPI) startOptimisticBlockProcessor() { for opts := range api.optimisticBlockC { err := api.simulateBlock(opts) if err != nil { // Validation failed, mark the status of the builder as lowPrio (pessemistic). - newStatus := datastore.MakeBlockBuilderStatus(datastore.LowPrio) + newStatus := datastore.MakeBlockBuilderStatus(common.LowPrio) builderPubKey := opts.req.Message.BuilderPubkey.String() err := api.redis.SetBlockBuilderStatus(builderPubKey, newStatus) if err != nil { api.log.WithError(err).Error("could not set block builder status in redis") } - err = api.db.SetBlockBuilderStatus(builderPubKey, false /*isHighPrio*/, false /*isBlacklisted*/) + err = api.db.SetBlockBuilderStatus(builderPubKey, common.LowPrio) if err != nil { api.log.WithError(err).Error("could not set block builder status in database") } @@ -432,6 +452,18 @@ func (api *RelayAPI) startOptimisticBlockProcessor() { } } +// startProposerRefundProcessor keeps listening on the channel and issuing proposer refunds. +func (api *RelayAPI) startProposerRefundProcessor() { + for opts := range api.proposerRefundC { + // TODO(mikeneuder): Implement refund logic. + api.log.WithFields(logrus.Fields{ + "proposerPK": opts.proposerPK, + "builderPK": opts.builderPK, + "amount": opts.amount, + }).Info("Received proposer refund event.") + } +} + func (api *RelayAPI) processNewSlot(headSlot uint64) { _apiHeadSlot := api.headSlot.Load() if headSlot <= _apiHeadSlot { @@ -881,6 +913,30 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) if err != nil { log.WithError(err).Error("failed to increment builder-stats after getPayload") } + + simErr := api.simulateBlock(blockSimOptions{ + ctx: req.Context(), + log: log, + highPrio: true, // manually set to true for these blocks. + req: &BuilderBlockValidationRequest{ + BuilderSubmitBlockRequest: types.BuilderSubmitBlockRequest{ + Signature: payload.Signature, + Message: &bidTrace.BidTrace, + ExecutionPayload: getPayloadResp.Data, + }, + }, + }) + if simErr != nil { + log.WithError(err).Error("failed to simulate signed block") + api.proposerRefundC <- proposerRefundOptions{ + // Both PKs are hex strings. + proposerPK: proposerPubkey.String(), + builderPK: bidTrace.BuilderPubkey.PubkeyHex().String(), + amount: bidTrace.Value, + slot: slot, + bidTrace: bidTrace.BidTrace, + } + } }() // Publish the signed beacon block via beacon-node @@ -891,13 +947,6 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) } signedBeaconBlock := SignedBlindedBeaconBlockToBeaconBlock(payload, getPayloadResp.Data) _, _ = api.beaconClient.PublishBlock(signedBeaconBlock) // errors are logged inside. - // Analyze published blocks for invalid blocks aysnchronously. - // Plan: - // 1. Write blocks to an OptimisticPublishedBlocks channel. - // 2. A separate thread will read from the channel and distribute work - // to the validating nodes. - // 3. If any published block is determined to be invalid, we update - // the builder status to low-prio and calculate reimbursement. }() } @@ -1006,7 +1055,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque builderStatusCode, err := api.redis.GetBlockBuilderStatus(payload.Message.BuilderPubkey.String()) log = log.WithFields(logrus.Fields{ "builderStatus": builderStatusCode, - "builderStatusStr": builderStatusCode.String(), + "builderStatusStr": datastore.StatusStringFromCode(builderStatusCode), }) if err != nil { log.WithError(err).Error("could not get block builder status") @@ -1043,7 +1092,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque return } - if builderStatusCode == datastore.Blacklisted { + if builderStatusCode == common.Blacklisted { log.Info("builder is blacklisted") time.Sleep(200 * time.Millisecond) w.WriteHeader(http.StatusOK) @@ -1051,7 +1100,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } // In case only high-prio requests are accepted, fail others - if api.ffDisableLowPrioBuilders && builderStatusCode != datastore.HighPrio { + if api.ffDisableLowPrioBuilders && builderStatusCode != common.HighPrio { log.Info("rejecting low-prio builder (ff-disable-low-prio-builders)") time.Sleep(200 * time.Millisecond) w.WriteHeader(http.StatusOK) @@ -1060,7 +1109,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque log = log.WithFields(logrus.Fields{ "builderStatus": builderStatusCode, - "builderStatusStr": builderStatusCode.String(), + "builderStatusStr": datastore.StatusStringFromCode(builderStatusCode), "proposerPubkey": payload.Message.ProposerPubkey.String(), "parentHash": payload.Message.ParentHash.String(), "value": payload.Message.Value.String(), @@ -1130,7 +1179,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque // Construct simulation request. opts := blockSimOptions{ ctx: req.Context(), - highPrio: builderStatusCode == datastore.HighPrio, + highPrio: builderStatusCode == common.HighPrio, log: log, req: &BuilderBlockValidationRequest{ BuilderSubmitBlockRequest: *payload, @@ -1139,7 +1188,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } // Only perform simulation on hot path if we are in non-optimistic mode. - if builderStatusCode != datastore.Optimistic { + if builderStatusCode != common.Optimistic { simErr = api.simulateBlock(opts) if simErr != nil { api.RespondError(w, http.StatusBadRequest, simErr.Error()) @@ -1265,13 +1314,13 @@ func (api *RelayAPI) handleInternalBuilderStatus(w http.ResponseWriter, req *htt "isHighPrio": isHighPrio, "isBlacklisted": isBlacklisted, }).Info("updating builder status") - code := datastore.LowPrio + code := common.LowPrio if isBlacklisted { - code = datastore.Blacklisted + code = common.Blacklisted } else if isOptimistic { - code = datastore.Optimistic + code = common.Optimistic } else if isHighPrio { - code = datastore.HighPrio + code = common.HighPrio } newStatus := datastore.MakeBlockBuilderStatus(code) err := api.redis.SetBlockBuilderStatus(builderPubkey, newStatus) @@ -1279,7 +1328,7 @@ func (api *RelayAPI) handleInternalBuilderStatus(w http.ResponseWriter, req *htt api.log.WithError(err).Error("could not set block builder status in redis") } - err = api.db.SetBlockBuilderStatus(builderPubkey, isHighPrio, isBlacklisted) + err = api.db.SetBlockBuilderStatus(builderPubkey, code) if err != nil { api.log.WithError(err).Error("could not set block builder status in database") } diff --git a/services/housekeeper/housekeeper.go b/services/housekeeper/housekeeper.go index 6c58ef40..781648e7 100644 --- a/services/housekeeper/housekeeper.go +++ b/services/housekeeper/housekeeper.go @@ -338,14 +338,7 @@ func (hk *Housekeeper) updateBlockBuildersInRedis() { hk.log.Infof("updating %d block builders in Redis...", len(builders)) for _, builder := range builders { - code := datastore.LowPrio - if builder.IsOptimistic { - code = datastore.Optimistic - } else if builder.IsHighPrio { - code = datastore.HighPrio - } else if builder.IsBlacklisted { - code = datastore.Blacklisted - } + code := common.BlockBuilderStatusCode(builder.BuilderStatus) status := datastore.MakeBlockBuilderStatus(code) hk.log.Infof("updating block builder in Redis: %s - %s", builder.BuilderPubkey, status) err = hk.redis.SetBlockBuilderStatus(builder.BuilderPubkey, status) From 26ad10320b2668f37a451ccbaa6e710662b60f9e Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Sat, 31 Dec 2022 13:22:40 -0700 Subject: [PATCH 04/22] flow (c) update database with new table to persist refunds --- database/database.go | 28 +++++++++++++++++ database/migrations/001_init_database.go | 18 +++++++++-- database/mockdb.go | 4 +++ database/types.go | 15 +++++++++ database/vars/tables.go | 1 + services/api/service.go | 39 ++++-------------------- 6 files changed, 70 insertions(+), 35 deletions(-) diff --git a/database/database.go b/database/database.go index 9df54f55..9c49ef65 100644 --- a/database/database.go +++ b/database/database.go @@ -44,6 +44,8 @@ type IDatabaseService interface { SetBlockBuilderStatus(pubkey string, status common.BlockBuilderStatusCode) error UpsertBlockBuilderEntryAfterSubmission(lastSubmission *BuilderBlockSubmissionEntry, isError bool) error IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error + + SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock) error } type DatabaseService struct { @@ -486,3 +488,29 @@ func (s *DatabaseService) DeleteExecutionPayloads(idFirst, idLast uint64) error _, err := s.DB.Exec(query, idFirst, idLast) return err } + +func (s *DatabaseService) SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock) error { + _signedBlindedBeaconBlock, err := json.Marshal(signedBlindedBeaconBlock) + if err != nil { + return err + } + + validatorRefundEntry := ValidatorRefundEntry{ + SignedBlindedBeaconBlock: NewNullString(string(_signedBlindedBeaconBlock)), + + Slot: bidTrace.Slot, + Epoch: bidTrace.Slot / uint64(common.SlotsPerEpoch), + + BuilderPubkey: bidTrace.BuilderPubkey.String(), + ProposerPubkey: bidTrace.ProposerPubkey.String(), + + Value: bidTrace.Value.String(), + } + + query := `INSERT INTO ` + vars.TableValidatorRefunds + ` + (signed_blinded_beacon_block, slot, epoch, builder_pubkey, proposer_pubkey, value) VALUES + (:signed_blinded_beacon_block, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value) + ON CONFLICT DO NOTHING` + _, err = s.DB.NamedExec(query, validatorRefundEntry) + return err +} diff --git a/database/migrations/001_init_database.go b/database/migrations/001_init_database.go index 7710a5d9..4d918959 100644 --- a/database/migrations/001_init_database.go +++ b/database/migrations/001_init_database.go @@ -120,8 +120,7 @@ var Migration001InitDatabase = &migrate.Migration{ builder_pubkey varchar(98) NOT NULL, description text NOT NULL, - is_high_prio boolean NOT NULL, - is_blacklisted boolean NOT NULL, + builder_status bigint NOT NULL, last_submission_id bigint references ` + vars.TableBuilderBlockSubmission + `(id) on delete set null, last_submission_slot bigint NOT NULL, @@ -134,6 +133,21 @@ var Migration001InitDatabase = &migrate.Migration{ UNIQUE (builder_pubkey) ); + + CREATE TABLE IF NOT EXISTS ` + vars.TableValidatorRefunds + `( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + inserted_at timestamp NOT NULL default current_timestamp, + + signed_blinded_beacon_block json, + + epoch bigint NOT NULL, + slot bigint NOT NULL, + + builder_pubkey varchar(98) NOT NULL, + proposer_pubkey varchar(98) NOT NULL, + + value NUMERIC(48, 0), + ); `}, Down: []string{` DROP TABLE IF EXISTS ` + vars.TableBuilderBlockSubmission + `; diff --git a/database/mockdb.go b/database/mockdb.go index 4fb10fdb..d905ddfa 100644 --- a/database/mockdb.go +++ b/database/mockdb.go @@ -100,3 +100,7 @@ func (db MockDB) IncBlockBuilderStatsAfterGetHeader(slot uint64, blockhash strin func (db MockDB) IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error { return nil } + +func (db MockDB) SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock) error { + return nil +} diff --git a/database/types.go b/database/types.go index 2828b4fc..362347d4 100644 --- a/database/types.go +++ b/database/types.go @@ -198,3 +198,18 @@ type BlockBuilderEntry struct { NumSentGetPayload uint64 `db:"num_sent_getpayload" json:"num_sent_getpayload"` } + +type ValidatorRefundEntry struct { + ID int64 `db:"id"` + InsertedAt time.Time `db:"inserted_at"` + + SignedBlindedBeaconBlock sql.NullString `db:"signed_blinded_beacon_block"` + + Slot uint64 `db:"slot"` + Epoch uint64 `db:"epoch"` + + BuilderPubkey string `db:"builder_pubkey"` + ProposerPubkey string `db:"proposer_pubkey"` + + Value string `db:"value"` +} diff --git a/database/vars/tables.go b/database/vars/tables.go index 5e812b8c..769f6209 100644 --- a/database/vars/tables.go +++ b/database/vars/tables.go @@ -12,4 +12,5 @@ var ( TableBuilderBlockSubmission = tableBase + "_builder_block_submission" TableDeliveredPayload = tableBase + "_payload_delivered" TableBlockBuilder = tableBase + "_blockbuilder" + TableValidatorRefunds = tableBase + "_validator_refunds" ) diff --git a/services/api/service.go b/services/api/service.go index 04e8ed67..ac00f332 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -106,15 +106,6 @@ type blockSimOptions struct { req *BuilderBlockValidationRequest } -// Data needed to initiate a proposer refund. -type proposerRefundOptions struct { - proposerPK string - builderPK string - amount types.U256Str - slot uint64 - bidTrace types.BidTrace -} - // RelayAPI represents a single Relay instance type RelayAPI struct { opts RelayAPIOpts @@ -159,7 +150,6 @@ type RelayAPI struct { // Channel used to process optimistic blocks asynchronously. optimisticBlockC chan blockSimOptions - proposerRefundC chan proposerRefundOptions } // NewRelayAPI creates a new service. if builders is nil, allow any builder @@ -219,7 +209,6 @@ func NewRelayAPI(opts RelayAPIOpts) (api *RelayAPI, err error) { activeValidatorC: make(chan types.PubkeyHex, 450_000), validatorRegC: make(chan types.SignedValidatorRegistration, 450_000), optimisticBlockC: make(chan blockSimOptions, 450_000), - proposerRefundC: make(chan proposerRefundOptions, 450_000), } if os.Getenv("FORCE_GET_HEADER_204") == "1" { @@ -330,9 +319,6 @@ func (api *RelayAPI) StartServer() (err error) { for i := 0; i < numValidatorRegProcessors; i++ { go api.startValidatorRegistrationDBProcessor() } - - // TODO(mikeneuder): consider if we should use >1 proposerRefundProcessors. - go api.startProposerRefundProcessor() } // Process current slot @@ -452,18 +438,6 @@ func (api *RelayAPI) startOptimisticBlockProcessor() { } } -// startProposerRefundProcessor keeps listening on the channel and issuing proposer refunds. -func (api *RelayAPI) startProposerRefundProcessor() { - for opts := range api.proposerRefundC { - // TODO(mikeneuder): Implement refund logic. - api.log.WithFields(logrus.Fields{ - "proposerPK": opts.proposerPK, - "builderPK": opts.builderPK, - "amount": opts.amount, - }).Info("Received proposer refund event.") - } -} - func (api *RelayAPI) processNewSlot(headSlot uint64) { _apiHeadSlot := api.headSlot.Load() if headSlot <= _apiHeadSlot { @@ -928,13 +902,12 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) }) if simErr != nil { log.WithError(err).Error("failed to simulate signed block") - api.proposerRefundC <- proposerRefundOptions{ - // Both PKs are hex strings. - proposerPK: proposerPubkey.String(), - builderPK: bidTrace.BuilderPubkey.PubkeyHex().String(), - amount: bidTrace.Value, - slot: slot, - bidTrace: bidTrace.BidTrace, + err = api.db.SaveValidatorRefund(bidTrace, payload) + if err != nil { + log.WithError(err).WithFields(logrus.Fields{ + "bidTrace": bidTrace, + "signedBlindedBeaconBlock": payload, + }).Error("failed to save validator refund to database") } } }() From 3dddf4bc5cb4ddc18da4d052cdb3a529b69f0064 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Mon, 2 Jan 2023 11:51:39 -0700 Subject: [PATCH 05/22] adding collateral check and DB field --- database/database.go | 8 ++-- database/migrations/001_init_database.go | 4 +- database/types.go | 3 +- datastore/redis.go | 31 ++++++++++++---- services/api/service.go | 47 ++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 13 deletions(-) diff --git a/database/database.go b/database/database.go index 9c49ef65..d8aa5317 100644 --- a/database/database.go +++ b/database/database.go @@ -438,8 +438,8 @@ func (s *DatabaseService) UpsertBlockBuilderEntryAfterSubmission(lastSubmission // Upsert query := `INSERT INTO ` + vars.TableBlockBuilder + ` - (builder_pubkey, description, builder_status, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror) VALUES - (:builder_pubkey, :description, :builder_status, :last_submission_id, :last_submission_slot, :num_submissions_total, :num_submissions_simerror) + (builder_pubkey, description, builder_status, builder_collateral, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror) VALUES + (:builder_pubkey, :description, :builder_status, :builder_collateral, :last_submission_id, :last_submission_slot, :num_submissions_total, :num_submissions_simerror) ON CONFLICT (builder_pubkey) DO UPDATE SET last_submission_id = :last_submission_id, last_submission_slot = :last_submission_slot, @@ -450,14 +450,14 @@ func (s *DatabaseService) UpsertBlockBuilderEntryAfterSubmission(lastSubmission } func (s *DatabaseService) GetBlockBuilders() ([]*BlockBuilderEntry, error) { - query := `SELECT id, inserted_at, builder_pubkey, description, builder_status, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` ORDER BY id ASC;` + query := `SELECT id, inserted_at, builder_pubkey, description, builder_status, builder_collateral, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` ORDER BY id ASC;` entries := []*BlockBuilderEntry{} err := s.DB.Select(&entries, query) return entries, err } func (s *DatabaseService) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderEntry, error) { - query := `SELECT id, inserted_at, builder_pubkey, description, builder_status, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` WHERE builder_pubkey=$1;` + query := `SELECT id, inserted_at, builder_pubkey, description, builder_status, builder_collateral, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` WHERE builder_pubkey=$1;` entry := &BlockBuilderEntry{} err := s.DB.Get(entry, query, pubkey) return entry, err diff --git a/database/migrations/001_init_database.go b/database/migrations/001_init_database.go index 4d918959..32eee060 100644 --- a/database/migrations/001_init_database.go +++ b/database/migrations/001_init_database.go @@ -120,7 +120,8 @@ var Migration001InitDatabase = &migrate.Migration{ builder_pubkey varchar(98) NOT NULL, description text NOT NULL, - builder_status bigint NOT NULL, + builder_status bigint NOT NULL, + builder_collateral NUMERIC(48, 0), last_submission_id bigint references ` + vars.TableBuilderBlockSubmission + `(id) on delete set null, last_submission_slot bigint NOT NULL, @@ -155,6 +156,7 @@ var Migration001InitDatabase = &migrate.Migration{ DROP TABLE IF EXISTS ` + vars.TableBlockBuilder + `; DROP TABLE IF EXISTS ` + vars.TableExecutionPayload + `; DROP TABLE IF EXISTS ` + vars.TableValidatorRegistration + `; + DROP TABLE IF EXISTS ` + vars.TableValidatorRefunds + `; `}, DisableTransactionUp: false, DisableTransactionDown: false, diff --git a/database/types.go b/database/types.go index 362347d4..69b27b9a 100644 --- a/database/types.go +++ b/database/types.go @@ -188,7 +188,8 @@ type BlockBuilderEntry struct { BuilderPubkey string `db:"builder_pubkey" json:"builder_pubkey"` Description string `db:"description" json:"description"` - BuilderStatus uint8 `db:"builder_status" json:"builder_status"` + BuilderStatus uint8 `db:"builder_status" json:"builder_status"` + BuilderCollateral string `db:"builder_collateral" json:"builder_collateral"` LastSubmissionID sql.NullInt64 `db:"last_submission_id" json:"last_submission_id"` LastSubmissionSlot uint64 `db:"last_submission_slot" json:"last_submission_slot"` diff --git a/datastore/redis.go b/datastore/redis.go index 73daa6f2..f4c50d03 100644 --- a/datastore/redis.go +++ b/datastore/redis.go @@ -91,10 +91,11 @@ type RedisCache struct { keyKnownValidators string keyValidatorRegistrationTimestamp string - keyRelayConfig string - keyStats string - keyProposerDuties string - keyBlockBuilderStatus string + keyRelayConfig string + keyStats string + keyProposerDuties string + keyBlockBuilderStatus string + keyBlockBuilderCollateral string } func NewRedisCache(redisURI, prefix string) (*RedisCache, error) { @@ -119,9 +120,10 @@ func NewRedisCache(redisURI, prefix string) (*RedisCache, error) { keyValidatorRegistrationTimestamp: fmt.Sprintf("%s/%s:validator-registration-timestamp", redisPrefix, prefix), keyRelayConfig: fmt.Sprintf("%s/%s:relay-config", redisPrefix, prefix), - keyStats: fmt.Sprintf("%s/%s:stats", redisPrefix, prefix), - keyProposerDuties: fmt.Sprintf("%s/%s:proposer-duties", redisPrefix, prefix), - keyBlockBuilderStatus: fmt.Sprintf("%s/%s:block-builder-status", redisPrefix, prefix), + keyStats: fmt.Sprintf("%s/%s:stats", redisPrefix, prefix), + keyProposerDuties: fmt.Sprintf("%s/%s:proposer-duties", redisPrefix, prefix), + keyBlockBuilderStatus: fmt.Sprintf("%s/%s:block-builder-status", redisPrefix, prefix), + keyBlockBuilderCollateral: fmt.Sprintf("%s/%s:block-builder-collateral", redisPrefix, prefix), }, nil } @@ -358,6 +360,21 @@ func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (code common.Bl } } +func (r *RedisCache) SetBlockBuilderCollateral(builderPubkey, value string) (err error) { + return r.client.HSet(context.Background(), r.keyBlockBuilderCollateral, builderPubkey, value).Err() +} + +func (r *RedisCache) GetBlockBuilderCollateral(builderPubkey string) (value string, err error) { + res, err := r.client.HGet(context.Background(), r.keyBlockBuilderCollateral, builderPubkey).Result() + if errors.Is(err, redis.Nil) { + return "", nil + } + if err != nil { + return "", err + } + return res, nil +} + func (r *RedisCache) GetBuilderLatestPayloadReceivedAt(slot uint64, builderPubkey, parentHash, proposerPubkey string) (int64, error) { keyLatestBidsTime := r.keyBlockBuilderLatestBidsTime(slot, parentHash, proposerPubkey) timestamp, err := r.client.HGet(context.Background(), keyLatestBidsTime, builderPubkey).Int64() diff --git a/services/api/service.go b/services/api/service.go index ac00f332..d37f9160 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -1034,6 +1034,51 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque log.WithError(err).Error("could not get block builder status") } + // Check for collateral in optimistic case. + if builderStatusCode == common.Optimistic { + builderCollateralStr, err := api.redis.GetBlockBuilderCollateral(payload.Message.BuilderPubkey.String()) + if err != nil { + log.WithError(err).Error("could not get block builder collateral, setting status as low prio") + builderStatusCode = common.LowPrio + builderCollateralStr = "" + } + + // Try to parse builder collateral string (U256Str) type. + var builderCollateral types.U256Str + err = builderCollateral.UnmarshalText([]byte(builderCollateralStr)) + if err != nil { + log.WithError(err).Error("could parse builder collateral string, setting status as low prio") + builderStatusCode = common.LowPrio + builderCollateral = ZeroU256 + } + + // Check if builder collateral exceeds the value of the block. If not, set as just high prio instead of optimistic. + if builderCollateral.Cmp((*types.U256Str)(&payload.Message.Value)) < 0 { + builderStatusCode = common.HighPrio + } + + // Async update the builder status in redis and the db async if no longer optimistic. + go func() { + if builderStatusCode != common.Optimistic { + redisStatus := datastore.BlockBuilderStatus(datastore.StatusStringFromCode(builderStatusCode)) + err := api.redis.SetBlockBuilderStatus(payload.Message.BuilderPubkey.String(), redisStatus) + log = log.WithFields(logrus.Fields{ + "NewStatusCode": builderStatusCode, + }) + if err != nil { + log.WithError(err).Error("unable to update block builder status in redis") + } + + err = api.db.SetBlockBuilderStatus(payload.Message.BuilderPubkey.String(), builderStatusCode) + if err != nil { + log.WithError(err).Error("unable to update block builder status in the database") + } + + log.Info("builder status updated in redis and database.") + } + }() + } + // Timestamp check expectedTimestamp := api.genesisInfo.Data.GenesisTime + (payload.Message.Slot * 12) if payload.ExecutionPayload.Timestamp != expectedTimestamp { @@ -1167,6 +1212,8 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque api.RespondError(w, http.StatusBadRequest, simErr.Error()) } } else { + // Manually set to high priority for optimistic blocks. + opts.highPrio = true // Write optimistic block to channel for async validation. api.optimisticBlockC <- opts } From e7aa03255712266c1ea2bb051bf82f069df18c43 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Mon, 2 Jan 2023 13:34:43 -0700 Subject: [PATCH 06/22] remove db update --- services/api/service.go | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/services/api/service.go b/services/api/service.go index d37f9160..115d3283 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -1038,8 +1038,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque if builderStatusCode == common.Optimistic { builderCollateralStr, err := api.redis.GetBlockBuilderCollateral(payload.Message.BuilderPubkey.String()) if err != nil { - log.WithError(err).Error("could not get block builder collateral, setting status as low prio") - builderStatusCode = common.LowPrio + log.WithError(err).Error("could not get block builder collateral") builderCollateralStr = "" } @@ -1047,8 +1046,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque var builderCollateral types.U256Str err = builderCollateral.UnmarshalText([]byte(builderCollateralStr)) if err != nil { - log.WithError(err).Error("could parse builder collateral string, setting status as low prio") - builderStatusCode = common.LowPrio + log.WithError(err).Error("could parse builder collateral string") builderCollateral = ZeroU256 } @@ -1056,27 +1054,6 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque if builderCollateral.Cmp((*types.U256Str)(&payload.Message.Value)) < 0 { builderStatusCode = common.HighPrio } - - // Async update the builder status in redis and the db async if no longer optimistic. - go func() { - if builderStatusCode != common.Optimistic { - redisStatus := datastore.BlockBuilderStatus(datastore.StatusStringFromCode(builderStatusCode)) - err := api.redis.SetBlockBuilderStatus(payload.Message.BuilderPubkey.String(), redisStatus) - log = log.WithFields(logrus.Fields{ - "NewStatusCode": builderStatusCode, - }) - if err != nil { - log.WithError(err).Error("unable to update block builder status in redis") - } - - err = api.db.SetBlockBuilderStatus(payload.Message.BuilderPubkey.String(), builderStatusCode) - if err != nil { - log.WithError(err).Error("unable to update block builder status in the database") - } - - log.Info("builder status updated in redis and database.") - } - }() } // Timestamp check From 234ff04e241c5da5ceeba28a29a5bcfe11b5d97b Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Mon, 2 Jan 2023 14:28:46 -0700 Subject: [PATCH 07/22] add block builder demotion in the case of a simulation error after a block is published. --- services/api/service.go | 60 ++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/services/api/service.go b/services/api/service.go index 115d3283..54dc2925 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -422,6 +422,7 @@ func (api *RelayAPI) startOptimisticBlockProcessor() { for opts := range api.optimisticBlockC { err := api.simulateBlock(opts) if err != nil { + api.log.WithError(err).Error("block simualtion failed") // Validation failed, mark the status of the builder as lowPrio (pessemistic). newStatus := datastore.MakeBlockBuilderStatus(common.LowPrio) builderPubKey := opts.req.Message.BuilderPubkey.String() @@ -888,26 +889,47 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) log.WithError(err).Error("failed to increment builder-stats after getPayload") } - simErr := api.simulateBlock(blockSimOptions{ - ctx: req.Context(), - log: log, - highPrio: true, // manually set to true for these blocks. - req: &BuilderBlockValidationRequest{ - BuilderSubmitBlockRequest: types.BuilderSubmitBlockRequest{ - Signature: payload.Signature, - Message: &bidTrace.BidTrace, - ExecutionPayload: getPayloadResp.Data, + builderStatus, err := api.redis.GetBlockBuilderStatus(bidTrace.BuilderPubkey.String()) + if err != nil { + log.WithError(err).Error("failed to get block builder status") + } + + // Check if the block was valid in the optimistic case. + if builderStatus == common.Optimistic { + simErr := api.simulateBlock(blockSimOptions{ + ctx: req.Context(), + log: log, + highPrio: true, // manually set to true for these blocks. + req: &BuilderBlockValidationRequest{ + BuilderSubmitBlockRequest: types.BuilderSubmitBlockRequest{ + Signature: payload.Signature, + Message: &bidTrace.BidTrace, + ExecutionPayload: getPayloadResp.Data, + }, }, - }, - }) - if simErr != nil { - log.WithError(err).Error("failed to simulate signed block") - err = api.db.SaveValidatorRefund(bidTrace, payload) - if err != nil { - log.WithError(err).WithFields(logrus.Fields{ - "bidTrace": bidTrace, - "signedBlindedBeaconBlock": payload, - }).Error("failed to save validator refund to database") + }) + if simErr != nil { + log.WithError(err).Error("failed to simulate signed block") + err = api.db.SaveValidatorRefund(bidTrace, payload) + if err != nil { + log.WithError(err).WithFields(logrus.Fields{ + "bidTrace": bidTrace, + "signedBlindedBeaconBlock": payload, + }).Error("failed to save validator refund to database") + } + + // Validation failed, mark the status of the builder as lowPrio (pessemistic). + newStatus := datastore.MakeBlockBuilderStatus(common.LowPrio) + builderPubKey := bidTrace.BuilderPubkey.String() + err := api.redis.SetBlockBuilderStatus(builderPubKey, newStatus) + if err != nil { + api.log.WithError(err).Error("could not set block builder status in redis") + } + + err = api.db.SetBlockBuilderStatus(builderPubKey, common.LowPrio) + if err != nil { + api.log.WithError(err).Error("could not set block builder status in database") + } } } }() From c36ef85cdd030ab57e689c40a27053365f13c007 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Mon, 2 Jan 2023 15:17:18 -0700 Subject: [PATCH 08/22] comments from Justin --- common/common.go | 13 +++++++ datastore/redis.go | 39 ++++---------------- datastore/utils.go | 16 --------- services/api/service.go | 55 +++++++++++++---------------- services/housekeeper/housekeeper.go | 5 ++- 5 files changed, 46 insertions(+), 82 deletions(-) delete mode 100644 datastore/utils.go diff --git a/common/common.go b/common/common.go index 01966253..e7697989 100644 --- a/common/common.go +++ b/common/common.go @@ -30,3 +30,16 @@ const ( Optimistic Blacklisted ) + +func (b BlockBuilderStatusCode) String() string { + switch b { + case Optimistic: + return "optimistic" + case HighPrio: + return "high-prio" + case Blacklisted: + return "blacklisted" + default: + return "low-prio" + } +} diff --git a/datastore/redis.go b/datastore/redis.go index f4c50d03..2aa502f4 100644 --- a/datastore/redis.go +++ b/datastore/redis.go @@ -32,28 +32,6 @@ var ( ErrFailedUpdatingTopBidNoBids = errors.New("failed to update top bid because no bids were found") ) -type BlockBuilderStatus string - -var ( - RedisBlockBuilderStatusLowPrio BlockBuilderStatus = "" - RedisBlockBuilderStatusHighPrio BlockBuilderStatus = "high-prio" - RedisBlockBuilderStatusOptimistic BlockBuilderStatus = "optimistic" - RedisBlockBuilderStatusBlacklisted BlockBuilderStatus = "blacklisted" -) - -func StatusStringFromCode(b common.BlockBuilderStatusCode) string { - switch b { - case common.Optimistic: - return string(RedisBlockBuilderStatusOptimistic) - case common.HighPrio: - return string(RedisBlockBuilderStatusHighPrio) - case common.Blacklisted: - return string(RedisBlockBuilderStatusBlacklisted) - default: - return string(RedisBlockBuilderStatusLowPrio) - } -} - func PubkeyHexToLowerStr(pk types.PubkeyHex) string { return strings.ToLower(string(pk)) } @@ -339,8 +317,8 @@ func (r *RedisCache) GetBidTrace(slot uint64, proposerPubkey, blockHash string) return resp, err } -func (r *RedisCache) SetBlockBuilderStatus(builderPubkey string, status BlockBuilderStatus) (err error) { - return r.client.HSet(context.Background(), r.keyBlockBuilderStatus, builderPubkey, string(status)).Err() +func (r *RedisCache) SetBlockBuilderStatus(builderPubkey string, code common.BlockBuilderStatusCode) (err error) { + return r.client.HSet(context.Background(), r.keyBlockBuilderStatus, builderPubkey, code).Err() } func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (code common.BlockBuilderStatusCode, err error) { @@ -348,16 +326,11 @@ func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (code common.Bl if errors.Is(err, redis.Nil) { return code, nil } - switch status := BlockBuilderStatus(res); status { - case RedisBlockBuilderStatusHighPrio: - return common.HighPrio, nil - case RedisBlockBuilderStatusOptimistic: - return common.Optimistic, nil - case RedisBlockBuilderStatusBlacklisted: - return common.Blacklisted, nil - default: - return common.LowPrio, nil + in, err := strconv.Atoi(res) + if err != nil { + return code, err } + return common.BlockBuilderStatusCode(in), nil } func (r *RedisCache) SetBlockBuilderCollateral(builderPubkey, value string) (err error) { diff --git a/datastore/utils.go b/datastore/utils.go deleted file mode 100644 index 223e4d11..00000000 --- a/datastore/utils.go +++ /dev/null @@ -1,16 +0,0 @@ -package datastore - -import "github.com/flashbots/mev-boost-relay/common" - -func MakeBlockBuilderStatus(c common.BlockBuilderStatusCode) BlockBuilderStatus { - switch c { - case common.HighPrio: - return RedisBlockBuilderStatusHighPrio - case common.Optimistic: - return RedisBlockBuilderStatusOptimistic - case common.Blacklisted: - return RedisBlockBuilderStatusBlacklisted - default: - return RedisBlockBuilderStatusLowPrio - } -} diff --git a/services/api/service.go b/services/api/service.go index 54dc2925..d3329eeb 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -422,16 +422,15 @@ func (api *RelayAPI) startOptimisticBlockProcessor() { for opts := range api.optimisticBlockC { err := api.simulateBlock(opts) if err != nil { - api.log.WithError(err).Error("block simualtion failed") - // Validation failed, mark the status of the builder as lowPrio (pessemistic). - newStatus := datastore.MakeBlockBuilderStatus(common.LowPrio) - builderPubKey := opts.req.Message.BuilderPubkey.String() - err := api.redis.SetBlockBuilderStatus(builderPubKey, newStatus) + api.log.WithError(err).Error("block simulation failed") + builderPubkey := opts.req.Message.BuilderPubkey.String() + // Validation failed, mark the status of the builder as lowPrio (pessimistic). + err := api.redis.SetBlockBuilderStatus(builderPubkey, common.LowPrio) if err != nil { api.log.WithError(err).Error("could not set block builder status in redis") } - err = api.db.SetBlockBuilderStatus(builderPubKey, common.LowPrio) + err = api.db.SetBlockBuilderStatus(builderPubkey, common.LowPrio) if err != nil { api.log.WithError(err).Error("could not set block builder status in database") } @@ -918,15 +917,14 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) }).Error("failed to save validator refund to database") } - // Validation failed, mark the status of the builder as lowPrio (pessemistic). - newStatus := datastore.MakeBlockBuilderStatus(common.LowPrio) - builderPubKey := bidTrace.BuilderPubkey.String() - err := api.redis.SetBlockBuilderStatus(builderPubKey, newStatus) + builderPubkey := bidTrace.BuilderPubkey.String() + // Validation failed, mark the status of the builder as lowPrio (pessimistic). + err := api.redis.SetBlockBuilderStatus(builderPubkey, common.LowPrio) if err != nil { api.log.WithError(err).Error("could not set block builder status in redis") } - err = api.db.SetBlockBuilderStatus(builderPubKey, common.LowPrio) + err = api.db.SetBlockBuilderStatus(builderPubkey, common.LowPrio) if err != nil { api.log.WithError(err).Error("could not set block builder status in database") } @@ -1050,12 +1048,19 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque builderStatusCode, err := api.redis.GetBlockBuilderStatus(payload.Message.BuilderPubkey.String()) log = log.WithFields(logrus.Fields{ "builderStatus": builderStatusCode, - "builderStatusStr": datastore.StatusStringFromCode(builderStatusCode), + "builderStatusStr": builderStatusCode.String(), }) if err != nil { log.WithError(err).Error("could not get block builder status") } + if builderStatusCode == common.Blacklisted { + log.Info("builder is blacklisted") + time.Sleep(200 * time.Millisecond) + w.WriteHeader(http.StatusOK) + return + } + // Check for collateral in optimistic case. if builderStatusCode == common.Optimistic { builderCollateralStr, err := api.redis.GetBlockBuilderCollateral(payload.Message.BuilderPubkey.String()) @@ -1109,13 +1114,6 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque return } - if builderStatusCode == common.Blacklisted { - log.Info("builder is blacklisted") - time.Sleep(200 * time.Millisecond) - w.WriteHeader(http.StatusOK) - return - } - // In case only high-prio requests are accepted, fail others if api.ffDisableLowPrioBuilders && builderStatusCode != common.HighPrio { log.Info("rejecting low-prio builder (ff-disable-low-prio-builders)") @@ -1126,7 +1124,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque log = log.WithFields(logrus.Fields{ "builderStatus": builderStatusCode, - "builderStatusStr": datastore.StatusStringFromCode(builderStatusCode), + "builderStatusStr": builderStatusCode.String(), "proposerPubkey": payload.Message.ProposerPubkey.String(), "parentHash": payload.Message.ParentHash.String(), "value": payload.Message.Value.String(), @@ -1196,7 +1194,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque // Construct simulation request. opts := blockSimOptions{ ctx: req.Context(), - highPrio: builderStatusCode == common.HighPrio, + highPrio: builderStatusCode == common.HighPrio || builderStatusCode == common.Optimistic, log: log, req: &BuilderBlockValidationRequest{ BuilderSubmitBlockRequest: *payload, @@ -1205,16 +1203,14 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } // Only perform simulation on hot path if we are in non-optimistic mode. - if builderStatusCode != common.Optimistic { + if builderStatusCode == common.Optimistic { + // Write optimistic block to channel for async validation. + api.optimisticBlockC <- opts + } else { simErr = api.simulateBlock(opts) if simErr != nil { api.RespondError(w, http.StatusBadRequest, simErr.Error()) } - } else { - // Manually set to high priority for optimistic blocks. - opts.highPrio = true - // Write optimistic block to channel for async validation. - api.optimisticBlockC <- opts } // Ensure this request is still the latest one @@ -1341,8 +1337,7 @@ func (api *RelayAPI) handleInternalBuilderStatus(w http.ResponseWriter, req *htt } else if isHighPrio { code = common.HighPrio } - newStatus := datastore.MakeBlockBuilderStatus(code) - err := api.redis.SetBlockBuilderStatus(builderPubkey, newStatus) + err := api.redis.SetBlockBuilderStatus(builderPubkey, code) if err != nil { api.log.WithError(err).Error("could not set block builder status in redis") } @@ -1352,7 +1347,7 @@ func (api *RelayAPI) handleInternalBuilderStatus(w http.ResponseWriter, req *htt api.log.WithError(err).Error("could not set block builder status in database") } - api.RespondOK(w, struct{ newStatus string }{newStatus: string(newStatus)}) + api.RespondOK(w, struct{ newStatus string }{newStatus: code.String()}) } } diff --git a/services/housekeeper/housekeeper.go b/services/housekeeper/housekeeper.go index 781648e7..8deb0072 100644 --- a/services/housekeeper/housekeeper.go +++ b/services/housekeeper/housekeeper.go @@ -339,9 +339,8 @@ func (hk *Housekeeper) updateBlockBuildersInRedis() { hk.log.Infof("updating %d block builders in Redis...", len(builders)) for _, builder := range builders { code := common.BlockBuilderStatusCode(builder.BuilderStatus) - status := datastore.MakeBlockBuilderStatus(code) - hk.log.Infof("updating block builder in Redis: %s - %s", builder.BuilderPubkey, status) - err = hk.redis.SetBlockBuilderStatus(builder.BuilderPubkey, status) + hk.log.Infof("updating block builder in Redis: %s - %s", builder.BuilderPubkey, code.String()) + err = hk.redis.SetBlockBuilderStatus(builder.BuilderPubkey, code) if err != nil { hk.log.WithError(err).Error("failed to set block builder status in redis") } From 8379b4e3a078b84c78961d99e0bc7b3818007c8d Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Mon, 2 Jan 2023 15:52:33 -0700 Subject: [PATCH 09/22] adding signedValidatorRegistration to the refund table --- database/database.go | 19 ++++++++++---- database/migrations/001_init_database.go | 6 ++++- database/mockdb.go | 2 +- database/types.go | 6 ++++- services/api/service.go | 33 +++++++++++++++++------- 5 files changed, 48 insertions(+), 18 deletions(-) diff --git a/database/database.go b/database/database.go index d8aa5317..32533868 100644 --- a/database/database.go +++ b/database/database.go @@ -45,7 +45,7 @@ type IDatabaseService interface { UpsertBlockBuilderEntryAfterSubmission(lastSubmission *BuilderBlockSubmissionEntry, isError bool) error IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error - SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock) error + SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error } type DatabaseService struct { @@ -489,14 +489,20 @@ func (s *DatabaseService) DeleteExecutionPayloads(idFirst, idLast uint64) error return err } -func (s *DatabaseService) SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock) error { +func (s *DatabaseService) SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { _signedBlindedBeaconBlock, err := json.Marshal(signedBlindedBeaconBlock) if err != nil { return err } + _signedValidatorRegistration, err := json.Marshal(signedValidatorRegistration) + if err != nil { + return err + } + validatorRefundEntry := ValidatorRefundEntry{ - SignedBlindedBeaconBlock: NewNullString(string(_signedBlindedBeaconBlock)), + SignedBlindedBeaconBlock: NewNullString(string(_signedBlindedBeaconBlock)), + SignedValidatorRegistration: NewNullString(string(_signedValidatorRegistration)), Slot: bidTrace.Slot, Epoch: bidTrace.Slot / uint64(common.SlotsPerEpoch), @@ -505,11 +511,14 @@ func (s *DatabaseService) SaveValidatorRefund(bidTrace *common.BidTraceV2, signe ProposerPubkey: bidTrace.ProposerPubkey.String(), Value: bidTrace.Value.String(), + + FeeRecipient: signedValidatorRegistration.Message.FeeRecipient.String(), + GasLimit: signedValidatorRegistration.Message.GasLimit, } query := `INSERT INTO ` + vars.TableValidatorRefunds + ` - (signed_blinded_beacon_block, slot, epoch, builder_pubkey, proposer_pubkey, value) VALUES - (:signed_blinded_beacon_block, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value) + (signed_blinded_beacon_block, signed_validator_registration, slot, epoch, builder_pubkey, proposer_pubkey, value, fee_recipient, gas_limit) VALUES + (:signed_blinded_beacon_block, :signed_validator_registration, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value, :fee_recipient, :gas_limit) ON CONFLICT DO NOTHING` _, err = s.DB.NamedExec(query, validatorRefundEntry) return err diff --git a/database/migrations/001_init_database.go b/database/migrations/001_init_database.go index 32eee060..9c21b6b7 100644 --- a/database/migrations/001_init_database.go +++ b/database/migrations/001_init_database.go @@ -139,7 +139,8 @@ var Migration001InitDatabase = &migrate.Migration{ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, inserted_at timestamp NOT NULL default current_timestamp, - signed_blinded_beacon_block json, + signed_blinded_beacon_block json, + signed_validator_registration json, epoch bigint NOT NULL, slot bigint NOT NULL, @@ -148,6 +149,9 @@ var Migration001InitDatabase = &migrate.Migration{ proposer_pubkey varchar(98) NOT NULL, value NUMERIC(48, 0), + + fee_recipient varchar(42) NOT NULL, + gas_limit bigint NOT NULL, ); `}, Down: []string{` diff --git a/database/mockdb.go b/database/mockdb.go index d905ddfa..05dd8ac1 100644 --- a/database/mockdb.go +++ b/database/mockdb.go @@ -101,6 +101,6 @@ func (db MockDB) IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error return nil } -func (db MockDB) SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock) error { +func (db MockDB) SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { return nil } diff --git a/database/types.go b/database/types.go index 69b27b9a..4e06fd25 100644 --- a/database/types.go +++ b/database/types.go @@ -204,7 +204,8 @@ type ValidatorRefundEntry struct { ID int64 `db:"id"` InsertedAt time.Time `db:"inserted_at"` - SignedBlindedBeaconBlock sql.NullString `db:"signed_blinded_beacon_block"` + SignedBlindedBeaconBlock sql.NullString `db:"signed_blinded_beacon_block"` + SignedValidatorRegistration sql.NullString `db:"signed_validator_registration"` Slot uint64 `db:"slot"` Epoch uint64 `db:"epoch"` @@ -213,4 +214,7 @@ type ValidatorRefundEntry struct { ProposerPubkey string `db:"proposer_pubkey"` Value string `db:"value"` + + FeeRecipient string `db:"fee_recipient"` + GasLimit uint64 `db:"gas_limit"` } diff --git a/services/api/service.go b/services/api/service.go index d3329eeb..3b0f9e28 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -909,25 +909,38 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) }) if simErr != nil { log.WithError(err).Error("failed to simulate signed block") - err = api.db.SaveValidatorRefund(bidTrace, payload) - if err != nil { - log.WithError(err).WithFields(logrus.Fields{ - "bidTrace": bidTrace, - "signedBlindedBeaconBlock": payload, - }).Error("failed to save validator refund to database") - } - builderPubkey := bidTrace.BuilderPubkey.String() // Validation failed, mark the status of the builder as lowPrio (pessimistic). - err := api.redis.SetBlockBuilderStatus(builderPubkey, common.LowPrio) + err = api.redis.SetBlockBuilderStatus(builderPubkey, common.LowPrio) if err != nil { api.log.WithError(err).Error("could not set block builder status in redis") } - err = api.db.SetBlockBuilderStatus(builderPubkey, common.LowPrio) if err != nil { api.log.WithError(err).Error("could not set block builder status in database") } + + registrationEntry, err := api.db.GetValidatorRegistration(bidTrace.ProposerPubkey.String()) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + api.log.WithError(err).Error("no validator registration found") + } else { + api.log.WithError(err).Error("error getting validator registration") + } + } + + signedRegistration, err := registrationEntry.ToSignedValidatorRegistration() + if err != nil { + api.log.WithError(err).Error("error converting registration entry to signed validator registration") + } + err = api.db.SaveValidatorRefund(bidTrace, payload, signedRegistration) + if err != nil { + log.WithError(err).WithFields(logrus.Fields{ + "bidTrace": bidTrace, + "signedBlindedBeaconBlock": payload, + "signedValidatorRegistration": signedRegistration, + }).Error("failed to save validator refund to database") + } } } }() From 442a7bf49f6e7ad4416d762b8df5365ff376d24d Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Mon, 2 Jan 2023 16:26:06 -0700 Subject: [PATCH 10/22] normalize usage of BuilderStatus and builder_status instead of BuilderStatusCode and other variants --- common/common.go | 6 +++--- database/database.go | 6 +++--- database/migrations/001_init_database.go | 4 ++-- database/mockdb.go | 2 +- database/types.go | 2 +- datastore/redis.go | 12 ++++++------ services/api/service.go | 22 +++++++++++----------- services/housekeeper/housekeeper.go | 2 +- 8 files changed, 28 insertions(+), 28 deletions(-) diff --git a/common/common.go b/common/common.go index e7697989..5f929501 100644 --- a/common/common.go +++ b/common/common.go @@ -22,16 +22,16 @@ type HTTPServerTimeouts struct { Idle time.Duration // Timeout to disconnect idle client connections. None if 0. } -type BlockBuilderStatusCode uint8 +type BlockBuilderStatus uint8 const ( - LowPrio BlockBuilderStatusCode = iota + LowPrio BlockBuilderStatus = iota HighPrio Optimistic Blacklisted ) -func (b BlockBuilderStatusCode) String() string { +func (b BlockBuilderStatus) String() string { switch b { case Optimistic: return "optimistic" diff --git a/database/database.go b/database/database.go index 32533868..19bb8321 100644 --- a/database/database.go +++ b/database/database.go @@ -41,7 +41,7 @@ type IDatabaseService interface { GetBlockBuilders() ([]*BlockBuilderEntry, error) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderEntry, error) - SetBlockBuilderStatus(pubkey string, status common.BlockBuilderStatusCode) error + SetBlockBuilderStatus(pubkey string, builderStatus common.BlockBuilderStatus) error UpsertBlockBuilderEntryAfterSubmission(lastSubmission *BuilderBlockSubmissionEntry, isError bool) error IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error @@ -463,9 +463,9 @@ func (s *DatabaseService) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderE return entry, err } -func (s *DatabaseService) SetBlockBuilderStatus(pubkey string, builderStatus common.BlockBuilderStatusCode) error { +func (s *DatabaseService) SetBlockBuilderStatus(pubkey string, builderStatusCode common.BlockBuilderStatus) error { query := `UPDATE ` + vars.TableBlockBuilder + ` SET builder_status=$1 WHERE builder_pubkey=$3;` - _, err := s.DB.Exec(query, uint8(builderStatus), pubkey) + _, err := s.DB.Exec(query, uint8(builderStatusCode), pubkey) return err } diff --git a/database/migrations/001_init_database.go b/database/migrations/001_init_database.go index 9c21b6b7..928a3b05 100644 --- a/database/migrations/001_init_database.go +++ b/database/migrations/001_init_database.go @@ -120,8 +120,8 @@ var Migration001InitDatabase = &migrate.Migration{ builder_pubkey varchar(98) NOT NULL, description text NOT NULL, - builder_status bigint NOT NULL, - builder_collateral NUMERIC(48, 0), + builder_status_code bigint NOT NULL, + builder_collateral NUMERIC(48, 0), last_submission_id bigint references ` + vars.TableBuilderBlockSubmission + `(id) on delete set null, last_submission_slot bigint NOT NULL, diff --git a/database/mockdb.go b/database/mockdb.go index 05dd8ac1..17f99a0a 100644 --- a/database/mockdb.go +++ b/database/mockdb.go @@ -89,7 +89,7 @@ func (db MockDB) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderEntry, err return nil, nil } -func (db MockDB) SetBlockBuilderStatus(pubkey string, builderStatus common.BlockBuilderStatusCode) error { +func (db MockDB) SetBlockBuilderStatus(pubkey string, builderStatus common.BlockBuilderStatus) error { return nil } diff --git a/database/types.go b/database/types.go index 4e06fd25..761819d1 100644 --- a/database/types.go +++ b/database/types.go @@ -188,7 +188,7 @@ type BlockBuilderEntry struct { BuilderPubkey string `db:"builder_pubkey" json:"builder_pubkey"` Description string `db:"description" json:"description"` - BuilderStatus uint8 `db:"builder_status" json:"builder_status"` + BuilderStatus uint8 `db:"builder_status" json:"builder_status"` BuilderCollateral string `db:"builder_collateral" json:"builder_collateral"` LastSubmissionID sql.NullInt64 `db:"last_submission_id" json:"last_submission_id"` diff --git a/datastore/redis.go b/datastore/redis.go index 2aa502f4..2fb60499 100644 --- a/datastore/redis.go +++ b/datastore/redis.go @@ -317,20 +317,20 @@ func (r *RedisCache) GetBidTrace(slot uint64, proposerPubkey, blockHash string) return resp, err } -func (r *RedisCache) SetBlockBuilderStatus(builderPubkey string, code common.BlockBuilderStatusCode) (err error) { - return r.client.HSet(context.Background(), r.keyBlockBuilderStatus, builderPubkey, code).Err() +func (r *RedisCache) SetBlockBuilderStatus(builderPubkey string, status common.BlockBuilderStatus) (err error) { + return r.client.HSet(context.Background(), r.keyBlockBuilderStatus, builderPubkey, status).Err() } -func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (code common.BlockBuilderStatusCode, err error) { +func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (status common.BlockBuilderStatus, err error) { res, err := r.client.HGet(context.Background(), r.keyBlockBuilderStatus, builderPubkey).Result() if errors.Is(err, redis.Nil) { - return code, nil + return status, nil } in, err := strconv.Atoi(res) if err != nil { - return code, err + return status, err } - return common.BlockBuilderStatusCode(in), nil + return common.BlockBuilderStatus(in), nil } func (r *RedisCache) SetBlockBuilderCollateral(builderPubkey, value string) (err error) { diff --git a/services/api/service.go b/services/api/service.go index 3b0f9e28..303e79e6 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -1058,16 +1058,16 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } } - builderStatusCode, err := api.redis.GetBlockBuilderStatus(payload.Message.BuilderPubkey.String()) + builderStatus, err := api.redis.GetBlockBuilderStatus(payload.Message.BuilderPubkey.String()) log = log.WithFields(logrus.Fields{ - "builderStatus": builderStatusCode, - "builderStatusStr": builderStatusCode.String(), + "builderStatus": builderStatus, + "builderStatusStr": builderStatus.String(), }) if err != nil { log.WithError(err).Error("could not get block builder status") } - if builderStatusCode == common.Blacklisted { + if builderStatus == common.Blacklisted { log.Info("builder is blacklisted") time.Sleep(200 * time.Millisecond) w.WriteHeader(http.StatusOK) @@ -1075,7 +1075,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } // Check for collateral in optimistic case. - if builderStatusCode == common.Optimistic { + if builderStatus == common.Optimistic { builderCollateralStr, err := api.redis.GetBlockBuilderCollateral(payload.Message.BuilderPubkey.String()) if err != nil { log.WithError(err).Error("could not get block builder collateral") @@ -1092,7 +1092,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque // Check if builder collateral exceeds the value of the block. If not, set as just high prio instead of optimistic. if builderCollateral.Cmp((*types.U256Str)(&payload.Message.Value)) < 0 { - builderStatusCode = common.HighPrio + builderStatus = common.HighPrio } } @@ -1128,7 +1128,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } // In case only high-prio requests are accepted, fail others - if api.ffDisableLowPrioBuilders && builderStatusCode != common.HighPrio { + if api.ffDisableLowPrioBuilders && builderStatus != common.HighPrio { log.Info("rejecting low-prio builder (ff-disable-low-prio-builders)") time.Sleep(200 * time.Millisecond) w.WriteHeader(http.StatusOK) @@ -1136,8 +1136,8 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } log = log.WithFields(logrus.Fields{ - "builderStatus": builderStatusCode, - "builderStatusStr": builderStatusCode.String(), + "builderStatus": builderStatus, + "builderStatusStr": builderStatus.String(), "proposerPubkey": payload.Message.ProposerPubkey.String(), "parentHash": payload.Message.ParentHash.String(), "value": payload.Message.Value.String(), @@ -1207,7 +1207,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque // Construct simulation request. opts := blockSimOptions{ ctx: req.Context(), - highPrio: builderStatusCode == common.HighPrio || builderStatusCode == common.Optimistic, + highPrio: builderStatus == common.HighPrio || builderStatus == common.Optimistic, log: log, req: &BuilderBlockValidationRequest{ BuilderSubmitBlockRequest: *payload, @@ -1216,7 +1216,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } // Only perform simulation on hot path if we are in non-optimistic mode. - if builderStatusCode == common.Optimistic { + if builderStatus == common.Optimistic { // Write optimistic block to channel for async validation. api.optimisticBlockC <- opts } else { diff --git a/services/housekeeper/housekeeper.go b/services/housekeeper/housekeeper.go index 8deb0072..57e2e421 100644 --- a/services/housekeeper/housekeeper.go +++ b/services/housekeeper/housekeeper.go @@ -338,7 +338,7 @@ func (hk *Housekeeper) updateBlockBuildersInRedis() { hk.log.Infof("updating %d block builders in Redis...", len(builders)) for _, builder := range builders { - code := common.BlockBuilderStatusCode(builder.BuilderStatus) + code := common.BlockBuilderStatus(builder.BuilderStatus) hk.log.Infof("updating block builder in Redis: %s - %s", builder.BuilderPubkey, code.String()) err = hk.redis.SetBlockBuilderStatus(builder.BuilderPubkey, code) if err != nil { From c04edc8a190b0db8f532a521dece89865ed0ad98 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Wed, 4 Jan 2023 12:33:31 -0700 Subject: [PATCH 11/22] rename refund to demotion --- database/database.go | 63 +++++++++++++++++------- database/migrations/001_init_database.go | 7 ++- database/mockdb.go | 2 +- database/types.go | 5 +- database/vars/tables.go | 2 +- services/api/service.go | 28 +++++++---- 6 files changed, 75 insertions(+), 32 deletions(-) diff --git a/database/database.go b/database/database.go index 19bb8321..fa11558e 100644 --- a/database/database.go +++ b/database/database.go @@ -45,7 +45,7 @@ type IDatabaseService interface { UpsertBlockBuilderEntryAfterSubmission(lastSubmission *BuilderBlockSubmissionEntry, isError bool) error IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error - SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error + UpsertBuilderDemotion(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error } type DatabaseService struct { @@ -489,20 +489,17 @@ func (s *DatabaseService) DeleteExecutionPayloads(idFirst, idLast uint64) error return err } -func (s *DatabaseService) SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { - _signedBlindedBeaconBlock, err := json.Marshal(signedBlindedBeaconBlock) - if err != nil { - return err +func (s *DatabaseService) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { + if bidTrace == nil { + return fmt.Errorf("nil bidTrace invalid for UpsertBuilderDemotion") } - _signedValidatorRegistration, err := json.Marshal(signedValidatorRegistration) + _bidTrace, err := json.Marshal(bidTrace) if err != nil { return err } - - validatorRefundEntry := ValidatorRefundEntry{ - SignedBlindedBeaconBlock: NewNullString(string(_signedBlindedBeaconBlock)), - SignedValidatorRegistration: NewNullString(string(_signedValidatorRegistration)), + builderDemotionEntry := BuilderDemotionEntry{ + UnsignedBidTrace: NewNullString(string(_bidTrace)), Slot: bidTrace.Slot, Epoch: bidTrace.Slot / uint64(common.SlotsPerEpoch), @@ -512,14 +509,46 @@ func (s *DatabaseService) SaveValidatorRefund(bidTrace *common.BidTraceV2, signe Value: bidTrace.Value.String(), - FeeRecipient: signedValidatorRegistration.Message.FeeRecipient.String(), - GasLimit: signedValidatorRegistration.Message.GasLimit, + BlockHash: bidTrace.BlockHash.String(), } - query := `INSERT INTO ` + vars.TableValidatorRefunds + ` - (signed_blinded_beacon_block, signed_validator_registration, slot, epoch, builder_pubkey, proposer_pubkey, value, fee_recipient, gas_limit) VALUES - (:signed_blinded_beacon_block, :signed_validator_registration, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value, :fee_recipient, :gas_limit) - ON CONFLICT DO NOTHING` - _, err = s.DB.NamedExec(query, validatorRefundEntry) + if signedBlindedBeaconBlock != nil { + _signedBlindedBeaconBlock, err := json.Marshal(signedBlindedBeaconBlock) + if err != nil { + return err + } + builderDemotionEntry.SignedBlindedBeaconBlock = NewNullString(string(_signedBlindedBeaconBlock)) + } + + if signedValidatorRegistration != nil { + _signedValidatorRegistration, err := json.Marshal(signedValidatorRegistration) + if err != nil { + return err + } + builderDemotionEntry.SignedValidatorRegistration = NewNullString(string(_signedValidatorRegistration)) + builderDemotionEntry.FeeRecipient = signedValidatorRegistration.Message.FeeRecipient.String() + builderDemotionEntry.GasLimit = signedValidatorRegistration.Message.GasLimit + } + + var query string + // If block_hash conflicts and we have a published block, fill in fields needed for the refund. + if signedBlindedBeaconBlock != nil && signedValidatorRegistration != nil { + query = `INSERT INTO ` + vars.TableBuidlerDemotions + ` + (unsigned_bid_trace, signed_blinded_beacon_block, signed_validator_registration, slot, epoch, builder_pubkey, proposer_pubkey, value, fee_recipient, gas_limit, block_hash) VALUES + (:unsigned_bid_trace, :signed_blinded_beacon_block, :signed_validator_registration, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value, :fee_recipient, :gas_limit, :block_hash) + ON CONFLICT (block_hash) DO UPDATE SET + signed_blinded_beacon_block = :signed_blinded_beacon_block, + signed_validator_registration = :signed_validator_registration, + fee_recipient = :fee_recipient, + gas_limit = :gas_limit; + ` + } else { + // If the block_hash conflicts, then all the relevant data must be there already. + query = `INSERT INTO ` + vars.TableBuidlerDemotions + ` + (unsigned_bid_trace, signed_blinded_beacon_block, signed_validator_registration, slot, epoch, builder_pubkey, proposer_pubkey, value, fee_recipient, gas_limit, block_hash) VALUES + (:unsigned_bid_trace, :signed_blinded_beacon_block, :signed_validator_registration, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value, :fee_recipient, :gas_limit, :block_hash) + ON CONFLICT (block_hash) DO NOTHING` + } + _, err = s.DB.NamedExec(query, builderDemotionEntry) return err } diff --git a/database/migrations/001_init_database.go b/database/migrations/001_init_database.go index 928a3b05..a685592e 100644 --- a/database/migrations/001_init_database.go +++ b/database/migrations/001_init_database.go @@ -135,10 +135,11 @@ var Migration001InitDatabase = &migrate.Migration{ UNIQUE (builder_pubkey) ); - CREATE TABLE IF NOT EXISTS ` + vars.TableValidatorRefunds + `( + CREATE TABLE IF NOT EXISTS ` + vars.TableBuidlerDemotions + `( id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, inserted_at timestamp NOT NULL default current_timestamp, + unsigned_bid_trace json, signed_blinded_beacon_block json, signed_validator_registration json, @@ -152,6 +153,8 @@ var Migration001InitDatabase = &migrate.Migration{ fee_recipient varchar(42) NOT NULL, gas_limit bigint NOT NULL, + + block_hash varchar(66) NOT NULL, ); `}, Down: []string{` @@ -160,7 +163,7 @@ var Migration001InitDatabase = &migrate.Migration{ DROP TABLE IF EXISTS ` + vars.TableBlockBuilder + `; DROP TABLE IF EXISTS ` + vars.TableExecutionPayload + `; DROP TABLE IF EXISTS ` + vars.TableValidatorRegistration + `; - DROP TABLE IF EXISTS ` + vars.TableValidatorRefunds + `; + DROP TABLE IF EXISTS ` + vars.TableBuidlerDemotions + `; `}, DisableTransactionUp: false, DisableTransactionDown: false, diff --git a/database/mockdb.go b/database/mockdb.go index 17f99a0a..01db02be 100644 --- a/database/mockdb.go +++ b/database/mockdb.go @@ -101,6 +101,6 @@ func (db MockDB) IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error return nil } -func (db MockDB) SaveValidatorRefund(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { +func (db MockDB) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { return nil } diff --git a/database/types.go b/database/types.go index 761819d1..72713229 100644 --- a/database/types.go +++ b/database/types.go @@ -200,10 +200,11 @@ type BlockBuilderEntry struct { NumSentGetPayload uint64 `db:"num_sent_getpayload" json:"num_sent_getpayload"` } -type ValidatorRefundEntry struct { +type BuilderDemotionEntry struct { ID int64 `db:"id"` InsertedAt time.Time `db:"inserted_at"` + UnsignedBidTrace sql.NullString `db:"unsigned_bid_trace"` SignedBlindedBeaconBlock sql.NullString `db:"signed_blinded_beacon_block"` SignedValidatorRegistration sql.NullString `db:"signed_validator_registration"` @@ -217,4 +218,6 @@ type ValidatorRefundEntry struct { FeeRecipient string `db:"fee_recipient"` GasLimit uint64 `db:"gas_limit"` + + BlockHash string `db:"block_hash"` } diff --git a/database/vars/tables.go b/database/vars/tables.go index 769f6209..96cadfc1 100644 --- a/database/vars/tables.go +++ b/database/vars/tables.go @@ -12,5 +12,5 @@ var ( TableBuilderBlockSubmission = tableBase + "_builder_block_submission" TableDeliveredPayload = tableBase + "_payload_delivered" TableBlockBuilder = tableBase + "_blockbuilder" - TableValidatorRefunds = tableBase + "_validator_refunds" + TableBuidlerDemotions = tableBase + "_builder_demotions" ) diff --git a/services/api/service.go b/services/api/service.go index 303e79e6..8d5e8768 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -402,17 +402,14 @@ func (api *RelayAPI) startValidatorRegistrationDBProcessor() { func (api *RelayAPI) simulateBlock(opts blockSimOptions) error { t := time.Now() simErr := api.blockSimRateLimiter.send(opts.ctx, opts.req, opts.highPrio) + log := opts.log.WithFields(logrus.Fields{ + "duration": time.Since(t).Seconds(), + "numWaiting": api.blockSimRateLimiter.currentCounter(), + }) if simErr != nil { - log := opts.log.WithField("simErr", simErr.Error()) - log.WithError(simErr).WithFields(logrus.Fields{ - "duration": time.Since(t).Seconds(), - "numWaiting": api.blockSimRateLimiter.currentCounter(), - }).Info("block validation failed") + log.WithError(simErr).Error("block validation failed") } else { - opts.log.WithFields(logrus.Fields{ - "duration": time.Since(t).Seconds(), - "numWaiting": api.blockSimRateLimiter.currentCounter(), - }).Info("block validation successful") + log.Info("block validation successful") } return simErr } @@ -434,6 +431,17 @@ func (api *RelayAPI) startOptimisticBlockProcessor() { if err != nil { api.log.WithError(err).Error("could not set block builder status in database") } + + bidTrace := &common.BidTraceV2{ + BidTrace: *opts.req.Message, + } + // Upsert into the builder demotion table but without the + // blinded block or the validator registration, becuase we don't + // know if this bid will be accepted. + err = api.db.UpsertBuilderDemotion(bidTrace, nil, nil) + if err != nil { + api.log.WithError(err).Error("could not upsert bid trace") + } } } } @@ -933,7 +941,7 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) if err != nil { api.log.WithError(err).Error("error converting registration entry to signed validator registration") } - err = api.db.SaveValidatorRefund(bidTrace, payload, signedRegistration) + err = api.db.UpsertBuilderDemotion(bidTrace, payload, signedRegistration) if err != nil { log.WithError(err).WithFields(logrus.Fields{ "bidTrace": bidTrace, From 66b14f7a0515aaa283eebe7bc896d20e94963b42 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Wed, 4 Jan 2023 12:36:57 -0700 Subject: [PATCH 12/22] s/TableBuidlerDemotions/TableBuilderDemotions/g --- database/database.go | 4 ++-- database/migrations/001_init_database.go | 4 ++-- database/vars/tables.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/database/database.go b/database/database.go index fa11558e..2a4e516c 100644 --- a/database/database.go +++ b/database/database.go @@ -533,7 +533,7 @@ func (s *DatabaseService) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, sig var query string // If block_hash conflicts and we have a published block, fill in fields needed for the refund. if signedBlindedBeaconBlock != nil && signedValidatorRegistration != nil { - query = `INSERT INTO ` + vars.TableBuidlerDemotions + ` + query = `INSERT INTO ` + vars.TableBuilderDemotions + ` (unsigned_bid_trace, signed_blinded_beacon_block, signed_validator_registration, slot, epoch, builder_pubkey, proposer_pubkey, value, fee_recipient, gas_limit, block_hash) VALUES (:unsigned_bid_trace, :signed_blinded_beacon_block, :signed_validator_registration, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value, :fee_recipient, :gas_limit, :block_hash) ON CONFLICT (block_hash) DO UPDATE SET @@ -544,7 +544,7 @@ func (s *DatabaseService) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, sig ` } else { // If the block_hash conflicts, then all the relevant data must be there already. - query = `INSERT INTO ` + vars.TableBuidlerDemotions + ` + query = `INSERT INTO ` + vars.TableBuilderDemotions + ` (unsigned_bid_trace, signed_blinded_beacon_block, signed_validator_registration, slot, epoch, builder_pubkey, proposer_pubkey, value, fee_recipient, gas_limit, block_hash) VALUES (:unsigned_bid_trace, :signed_blinded_beacon_block, :signed_validator_registration, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value, :fee_recipient, :gas_limit, :block_hash) ON CONFLICT (block_hash) DO NOTHING` diff --git a/database/migrations/001_init_database.go b/database/migrations/001_init_database.go index a685592e..51598847 100644 --- a/database/migrations/001_init_database.go +++ b/database/migrations/001_init_database.go @@ -135,7 +135,7 @@ var Migration001InitDatabase = &migrate.Migration{ UNIQUE (builder_pubkey) ); - CREATE TABLE IF NOT EXISTS ` + vars.TableBuidlerDemotions + `( + CREATE TABLE IF NOT EXISTS ` + vars.TableBuilderDemotions + `( id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, inserted_at timestamp NOT NULL default current_timestamp, @@ -163,7 +163,7 @@ var Migration001InitDatabase = &migrate.Migration{ DROP TABLE IF EXISTS ` + vars.TableBlockBuilder + `; DROP TABLE IF EXISTS ` + vars.TableExecutionPayload + `; DROP TABLE IF EXISTS ` + vars.TableValidatorRegistration + `; - DROP TABLE IF EXISTS ` + vars.TableBuidlerDemotions + `; + DROP TABLE IF EXISTS ` + vars.TableBuilderDemotions + `; `}, DisableTransactionUp: false, DisableTransactionDown: false, diff --git a/database/vars/tables.go b/database/vars/tables.go index 96cadfc1..75466690 100644 --- a/database/vars/tables.go +++ b/database/vars/tables.go @@ -12,5 +12,5 @@ var ( TableBuilderBlockSubmission = tableBase + "_builder_block_submission" TableDeliveredPayload = tableBase + "_payload_delivered" TableBlockBuilder = tableBase + "_blockbuilder" - TableBuidlerDemotions = tableBase + "_builder_demotions" + TableBuilderDemotions = tableBase + "_builder_demotions" ) From 6bb8bf357d441c7228c431d16c8dc70027748fa8 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Wed, 4 Jan 2023 13:09:43 -0700 Subject: [PATCH 13/22] normalizing use of builder instead of block builder except in the case of interating with the block builder table specifically --- common/common.go | 6 ++--- database/database.go | 4 ++-- database/mockdb.go | 2 +- datastore/redis.go | 36 ++++++++++++++--------------- services/api/service.go | 20 ++++++++-------- services/housekeeper/housekeeper.go | 6 ++--- 6 files changed, 37 insertions(+), 37 deletions(-) diff --git a/common/common.go b/common/common.go index 5f929501..1fc3cade 100644 --- a/common/common.go +++ b/common/common.go @@ -22,16 +22,16 @@ type HTTPServerTimeouts struct { Idle time.Duration // Timeout to disconnect idle client connections. None if 0. } -type BlockBuilderStatus uint8 +type BuilderStatus uint8 const ( - LowPrio BlockBuilderStatus = iota + LowPrio BuilderStatus = iota HighPrio Optimistic Blacklisted ) -func (b BlockBuilderStatus) String() string { +func (b BuilderStatus) String() string { switch b { case Optimistic: return "optimistic" diff --git a/database/database.go b/database/database.go index 2a4e516c..c1d1bcc2 100644 --- a/database/database.go +++ b/database/database.go @@ -41,7 +41,7 @@ type IDatabaseService interface { GetBlockBuilders() ([]*BlockBuilderEntry, error) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderEntry, error) - SetBlockBuilderStatus(pubkey string, builderStatus common.BlockBuilderStatus) error + SetBlockBuilderStatus(pubkey string, builderStatus common.BuilderStatus) error UpsertBlockBuilderEntryAfterSubmission(lastSubmission *BuilderBlockSubmissionEntry, isError bool) error IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error @@ -463,7 +463,7 @@ func (s *DatabaseService) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderE return entry, err } -func (s *DatabaseService) SetBlockBuilderStatus(pubkey string, builderStatusCode common.BlockBuilderStatus) error { +func (s *DatabaseService) SetBlockBuilderStatus(pubkey string, builderStatusCode common.BuilderStatus) error { query := `UPDATE ` + vars.TableBlockBuilder + ` SET builder_status=$1 WHERE builder_pubkey=$3;` _, err := s.DB.Exec(query, uint8(builderStatusCode), pubkey) return err diff --git a/database/mockdb.go b/database/mockdb.go index 01db02be..de00cd83 100644 --- a/database/mockdb.go +++ b/database/mockdb.go @@ -89,7 +89,7 @@ func (db MockDB) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderEntry, err return nil, nil } -func (db MockDB) SetBlockBuilderStatus(pubkey string, builderStatus common.BlockBuilderStatus) error { +func (db MockDB) SetBlockBuilderStatus(pubkey string, builderStatus common.BuilderStatus) error { return nil } diff --git a/datastore/redis.go b/datastore/redis.go index 2fb60499..b0a37edc 100644 --- a/datastore/redis.go +++ b/datastore/redis.go @@ -69,11 +69,11 @@ type RedisCache struct { keyKnownValidators string keyValidatorRegistrationTimestamp string - keyRelayConfig string - keyStats string - keyProposerDuties string - keyBlockBuilderStatus string - keyBlockBuilderCollateral string + keyRelayConfig string + keyStats string + keyProposerDuties string + keyBuilderStatus string + keyBuilderCollateral string } func NewRedisCache(redisURI, prefix string) (*RedisCache, error) { @@ -98,10 +98,10 @@ func NewRedisCache(redisURI, prefix string) (*RedisCache, error) { keyValidatorRegistrationTimestamp: fmt.Sprintf("%s/%s:validator-registration-timestamp", redisPrefix, prefix), keyRelayConfig: fmt.Sprintf("%s/%s:relay-config", redisPrefix, prefix), - keyStats: fmt.Sprintf("%s/%s:stats", redisPrefix, prefix), - keyProposerDuties: fmt.Sprintf("%s/%s:proposer-duties", redisPrefix, prefix), - keyBlockBuilderStatus: fmt.Sprintf("%s/%s:block-builder-status", redisPrefix, prefix), - keyBlockBuilderCollateral: fmt.Sprintf("%s/%s:block-builder-collateral", redisPrefix, prefix), + keyStats: fmt.Sprintf("%s/%s:stats", redisPrefix, prefix), + keyProposerDuties: fmt.Sprintf("%s/%s:proposer-duties", redisPrefix, prefix), + keyBuilderStatus: fmt.Sprintf("%s/%s:block-builder-status", redisPrefix, prefix), + keyBuilderCollateral: fmt.Sprintf("%s/%s:block-builder-collateral", redisPrefix, prefix), }, nil } @@ -317,12 +317,12 @@ func (r *RedisCache) GetBidTrace(slot uint64, proposerPubkey, blockHash string) return resp, err } -func (r *RedisCache) SetBlockBuilderStatus(builderPubkey string, status common.BlockBuilderStatus) (err error) { - return r.client.HSet(context.Background(), r.keyBlockBuilderStatus, builderPubkey, status).Err() +func (r *RedisCache) SetBuilderStatus(builderPubkey string, status common.BuilderStatus) (err error) { + return r.client.HSet(context.Background(), r.keyBuilderStatus, builderPubkey, status).Err() } -func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (status common.BlockBuilderStatus, err error) { - res, err := r.client.HGet(context.Background(), r.keyBlockBuilderStatus, builderPubkey).Result() +func (r *RedisCache) GetBuilderStatus(builderPubkey string) (status common.BuilderStatus, err error) { + res, err := r.client.HGet(context.Background(), r.keyBuilderStatus, builderPubkey).Result() if errors.Is(err, redis.Nil) { return status, nil } @@ -330,15 +330,15 @@ func (r *RedisCache) GetBlockBuilderStatus(builderPubkey string) (status common. if err != nil { return status, err } - return common.BlockBuilderStatus(in), nil + return common.BuilderStatus(in), nil } -func (r *RedisCache) SetBlockBuilderCollateral(builderPubkey, value string) (err error) { - return r.client.HSet(context.Background(), r.keyBlockBuilderCollateral, builderPubkey, value).Err() +func (r *RedisCache) SetBuilderCollateral(builderPubkey, value string) (err error) { + return r.client.HSet(context.Background(), r.keyBuilderCollateral, builderPubkey, value).Err() } -func (r *RedisCache) GetBlockBuilderCollateral(builderPubkey string) (value string, err error) { - res, err := r.client.HGet(context.Background(), r.keyBlockBuilderCollateral, builderPubkey).Result() +func (r *RedisCache) GetBuilderCollateral(builderPubkey string) (value string, err error) { + res, err := r.client.HGet(context.Background(), r.keyBuilderCollateral, builderPubkey).Result() if errors.Is(err, redis.Nil) { return "", nil } diff --git a/services/api/service.go b/services/api/service.go index 8d5e8768..34097e6d 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -422,7 +422,7 @@ func (api *RelayAPI) startOptimisticBlockProcessor() { api.log.WithError(err).Error("block simulation failed") builderPubkey := opts.req.Message.BuilderPubkey.String() // Validation failed, mark the status of the builder as lowPrio (pessimistic). - err := api.redis.SetBlockBuilderStatus(builderPubkey, common.LowPrio) + err := api.redis.SetBuilderStatus(builderPubkey, common.LowPrio) if err != nil { api.log.WithError(err).Error("could not set block builder status in redis") } @@ -896,9 +896,9 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) log.WithError(err).Error("failed to increment builder-stats after getPayload") } - builderStatus, err := api.redis.GetBlockBuilderStatus(bidTrace.BuilderPubkey.String()) + builderStatus, err := api.redis.GetBuilderStatus(bidTrace.BuilderPubkey.String()) if err != nil { - log.WithError(err).Error("failed to get block builder status") + log.WithError(err).Error("failed to get builder status") } // Check if the block was valid in the optimistic case. @@ -919,9 +919,9 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) log.WithError(err).Error("failed to simulate signed block") builderPubkey := bidTrace.BuilderPubkey.String() // Validation failed, mark the status of the builder as lowPrio (pessimistic). - err = api.redis.SetBlockBuilderStatus(builderPubkey, common.LowPrio) + err = api.redis.SetBuilderStatus(builderPubkey, common.LowPrio) if err != nil { - api.log.WithError(err).Error("could not set block builder status in redis") + api.log.WithError(err).Error("could not set builder status in redis") } err = api.db.SetBlockBuilderStatus(builderPubkey, common.LowPrio) if err != nil { @@ -1066,7 +1066,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } } - builderStatus, err := api.redis.GetBlockBuilderStatus(payload.Message.BuilderPubkey.String()) + builderStatus, err := api.redis.GetBuilderStatus(payload.Message.BuilderPubkey.String()) log = log.WithFields(logrus.Fields{ "builderStatus": builderStatus, "builderStatusStr": builderStatus.String(), @@ -1084,7 +1084,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque // Check for collateral in optimistic case. if builderStatus == common.Optimistic { - builderCollateralStr, err := api.redis.GetBlockBuilderCollateral(payload.Message.BuilderPubkey.String()) + builderCollateralStr, err := api.redis.GetBuilderCollateral(payload.Message.BuilderPubkey.String()) if err != nil { log.WithError(err).Error("could not get block builder collateral") builderCollateralStr = "" @@ -1358,14 +1358,14 @@ func (api *RelayAPI) handleInternalBuilderStatus(w http.ResponseWriter, req *htt } else if isHighPrio { code = common.HighPrio } - err := api.redis.SetBlockBuilderStatus(builderPubkey, code) + err := api.redis.SetBuilderStatus(builderPubkey, code) if err != nil { - api.log.WithError(err).Error("could not set block builder status in redis") + api.log.WithError(err).Error("could not set builder status in redis") } err = api.db.SetBlockBuilderStatus(builderPubkey, code) if err != nil { - api.log.WithError(err).Error("could not set block builder status in database") + api.log.WithError(err).Error("could not set builder status in database") } api.RespondOK(w, struct{ newStatus string }{newStatus: code.String()}) diff --git a/services/housekeeper/housekeeper.go b/services/housekeeper/housekeeper.go index 57e2e421..b9223da6 100644 --- a/services/housekeeper/housekeeper.go +++ b/services/housekeeper/housekeeper.go @@ -338,11 +338,11 @@ func (hk *Housekeeper) updateBlockBuildersInRedis() { hk.log.Infof("updating %d block builders in Redis...", len(builders)) for _, builder := range builders { - code := common.BlockBuilderStatus(builder.BuilderStatus) + code := common.BuilderStatus(builder.BuilderStatus) hk.log.Infof("updating block builder in Redis: %s - %s", builder.BuilderPubkey, code.String()) - err = hk.redis.SetBlockBuilderStatus(builder.BuilderPubkey, code) + err = hk.redis.SetBuilderStatus(builder.BuilderPubkey, code) if err != nil { - hk.log.WithError(err).Error("failed to set block builder status in redis") + hk.log.WithError(err).Error("failed to set builder status in redis") } } } From 3a37075bbb2a02784358682e18f1eec01d87040d Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Wed, 4 Jan 2023 13:58:27 -0700 Subject: [PATCH 14/22] collateral id and demotions --- database/database.go | 18 +++++--- database/migrations/001_init_database.go | 5 ++- database/mockdb.go | 4 ++ database/types.go | 5 ++- services/api/service.go | 57 ++++++++++++++++-------- services/housekeeper/housekeeper.go | 2 +- 6 files changed, 62 insertions(+), 29 deletions(-) diff --git a/database/database.go b/database/database.go index c1d1bcc2..d966bf22 100644 --- a/database/database.go +++ b/database/database.go @@ -44,6 +44,7 @@ type IDatabaseService interface { SetBlockBuilderStatus(pubkey string, builderStatus common.BuilderStatus) error UpsertBlockBuilderEntryAfterSubmission(lastSubmission *BuilderBlockSubmissionEntry, isError bool) error IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error + GetBlockBuildersFromCollateralID(collateralID uint64) ([]*BlockBuilderEntry, error) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error } @@ -438,8 +439,8 @@ func (s *DatabaseService) UpsertBlockBuilderEntryAfterSubmission(lastSubmission // Upsert query := `INSERT INTO ` + vars.TableBlockBuilder + ` - (builder_pubkey, description, builder_status, builder_collateral, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror) VALUES - (:builder_pubkey, :description, :builder_status, :builder_collateral, :last_submission_id, :last_submission_slot, :num_submissions_total, :num_submissions_simerror) + (builder_pubkey, description, status, collateral_value, collateral_id, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror) VALUES + (:builder_pubkey, :description, :status, :collateral_value, :collateral_id, :last_submission_id, :last_submission_slot, :num_submissions_total, :num_submissions_simerror) ON CONFLICT (builder_pubkey) DO UPDATE SET last_submission_id = :last_submission_id, last_submission_slot = :last_submission_slot, @@ -450,21 +451,21 @@ func (s *DatabaseService) UpsertBlockBuilderEntryAfterSubmission(lastSubmission } func (s *DatabaseService) GetBlockBuilders() ([]*BlockBuilderEntry, error) { - query := `SELECT id, inserted_at, builder_pubkey, description, builder_status, builder_collateral, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` ORDER BY id ASC;` + query := `SELECT id, inserted_at, builder_pubkey, description, status, collateral_value, collateral_id, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` ORDER BY id ASC;` entries := []*BlockBuilderEntry{} err := s.DB.Select(&entries, query) return entries, err } func (s *DatabaseService) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderEntry, error) { - query := `SELECT id, inserted_at, builder_pubkey, description, builder_status, builder_collateral, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` WHERE builder_pubkey=$1;` + query := `SELECT id, inserted_at, builder_pubkey, description, status, collateral_value, collateral_id, last_submission_id, last_submission_slot, num_submissions_total, num_submissions_simerror, num_sent_getpayload FROM ` + vars.TableBlockBuilder + ` WHERE builder_pubkey=$1;` entry := &BlockBuilderEntry{} err := s.DB.Get(entry, query, pubkey) return entry, err } func (s *DatabaseService) SetBlockBuilderStatus(pubkey string, builderStatusCode common.BuilderStatus) error { - query := `UPDATE ` + vars.TableBlockBuilder + ` SET builder_status=$1 WHERE builder_pubkey=$3;` + query := `UPDATE ` + vars.TableBlockBuilder + ` SET status=$1 WHERE builder_pubkey=$3;` _, err := s.DB.Exec(query, uint8(builderStatusCode), pubkey) return err } @@ -552,3 +553,10 @@ func (s *DatabaseService) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, sig _, err = s.DB.NamedExec(query, builderDemotionEntry) return err } + +func (s *DatabaseService) GetBlockBuildersFromCollateralID(collateralID uint64) ([]*BlockBuilderEntry, error) { + query := `SELECT builder_pubkey FROM ` + vars.TableBlockBuilder + ` WHERE collateral_id=$1 ORDER BY id ASC;` + entries := []*BlockBuilderEntry{} + err := s.DB.Select(&entries, query, collateralID) + return entries, err +} diff --git a/database/migrations/001_init_database.go b/database/migrations/001_init_database.go index 51598847..91f790d8 100644 --- a/database/migrations/001_init_database.go +++ b/database/migrations/001_init_database.go @@ -120,8 +120,9 @@ var Migration001InitDatabase = &migrate.Migration{ builder_pubkey varchar(98) NOT NULL, description text NOT NULL, - builder_status_code bigint NOT NULL, - builder_collateral NUMERIC(48, 0), + status bigint NOT NULL, + collateral_value NUMERIC(48, 0), + collateral_id bigint NOT NULL, last_submission_id bigint references ` + vars.TableBuilderBlockSubmission + `(id) on delete set null, last_submission_slot bigint NOT NULL, diff --git a/database/mockdb.go b/database/mockdb.go index de00cd83..798fe9bf 100644 --- a/database/mockdb.go +++ b/database/mockdb.go @@ -104,3 +104,7 @@ func (db MockDB) IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error func (db MockDB) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { return nil } + +func (db MockDB) GetBlockBuildersFromCollateralID(collateralID uint64) ([]*BlockBuilderEntry, error) { + return nil, nil +} diff --git a/database/types.go b/database/types.go index 72713229..6348e7bc 100644 --- a/database/types.go +++ b/database/types.go @@ -188,8 +188,9 @@ type BlockBuilderEntry struct { BuilderPubkey string `db:"builder_pubkey" json:"builder_pubkey"` Description string `db:"description" json:"description"` - BuilderStatus uint8 `db:"builder_status" json:"builder_status"` - BuilderCollateral string `db:"builder_collateral" json:"builder_collateral"` + Status uint8 `db:"status" json:"status"` + Collateral string `db:"collateral_value" json:"collateral_value"` + CollateralID uint64 `db:"collateral_id" json:"collateral_id"` LastSubmissionID sql.NullInt64 `db:"last_submission_id" json:"last_submission_id"` LastSubmissionSlot uint64 `db:"last_submission_slot" json:"last_submission_slot"` diff --git a/services/api/service.go b/services/api/service.go index 34097e6d..2e488d8a 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -414,6 +414,40 @@ func (api *RelayAPI) simulateBlock(opts blockSimOptions) error { return simErr } +// Demotes a set of builders who have matching collateral IDs. This function +// continues to try demoting even if an error is encountered. +func (api *RelayAPI) demoteBuildersCollateralID(builderPubkey string) { + collateralPubkeys := []string{builderPubkey} + // Fetch builder collateral_id. + builder, err := api.db.GetBlockBuilderByPubkey(builderPubkey) + if err != nil { + api.log.WithError(err).Error("unable to get builder from database") + } + collateralID := builder.CollateralID + + // Fetch additional builder pubkeys using the collateral_id. + otherBuilders, err := api.db.GetBlockBuildersFromCollateralID(collateralID) + if err != nil { + api.log.WithError(err).Error("unable to get extra builders from collateral id") + } + for _, b := range otherBuilders { + collateralPubkeys = append(collateralPubkeys, b.BuilderPubkey) + } + + // Demote all the pubkeys in both redis and the db. + for _, pk := range collateralPubkeys { + err = api.redis.SetBuilderStatus(pk, common.LowPrio) + if err != nil { + api.log.WithError(err).Error("could not set builder status in redis") + } + + err = api.db.SetBlockBuilderStatus(pk, common.LowPrio) + if err != nil { + api.log.WithError(err).Error("could not set builder status in database") + } + } +} + // startOptimisticBlockProcessor keeps listening on the channel and validating incoming blocks asynchronously. func (api *RelayAPI) startOptimisticBlockProcessor() { for opts := range api.optimisticBlockC { @@ -421,16 +455,8 @@ func (api *RelayAPI) startOptimisticBlockProcessor() { if err != nil { api.log.WithError(err).Error("block simulation failed") builderPubkey := opts.req.Message.BuilderPubkey.String() - // Validation failed, mark the status of the builder as lowPrio (pessimistic). - err := api.redis.SetBuilderStatus(builderPubkey, common.LowPrio) - if err != nil { - api.log.WithError(err).Error("could not set block builder status in redis") - } - - err = api.db.SetBlockBuilderStatus(builderPubkey, common.LowPrio) - if err != nil { - api.log.WithError(err).Error("could not set block builder status in database") - } + // Validation failed, demote all builders with the collateral id. + api.demoteBuildersCollateralID(builderPubkey) bidTrace := &common.BidTraceV2{ BidTrace: *opts.req.Message, @@ -918,15 +944,8 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) if simErr != nil { log.WithError(err).Error("failed to simulate signed block") builderPubkey := bidTrace.BuilderPubkey.String() - // Validation failed, mark the status of the builder as lowPrio (pessimistic). - err = api.redis.SetBuilderStatus(builderPubkey, common.LowPrio) - if err != nil { - api.log.WithError(err).Error("could not set builder status in redis") - } - err = api.db.SetBlockBuilderStatus(builderPubkey, common.LowPrio) - if err != nil { - api.log.WithError(err).Error("could not set block builder status in database") - } + // Validation failed, demote all builders with the collateral id. + api.demoteBuildersCollateralID(builderPubkey) registrationEntry, err := api.db.GetValidatorRegistration(bidTrace.ProposerPubkey.String()) if err != nil { diff --git a/services/housekeeper/housekeeper.go b/services/housekeeper/housekeeper.go index b9223da6..f86b6151 100644 --- a/services/housekeeper/housekeeper.go +++ b/services/housekeeper/housekeeper.go @@ -338,7 +338,7 @@ func (hk *Housekeeper) updateBlockBuildersInRedis() { hk.log.Infof("updating %d block builders in Redis...", len(builders)) for _, builder := range builders { - code := common.BuilderStatus(builder.BuilderStatus) + code := common.BuilderStatus(builder.Status) hk.log.Infof("updating block builder in Redis: %s - %s", builder.BuilderPubkey, code.String()) err = hk.redis.SetBuilderStatus(builder.BuilderPubkey, code) if err != nil { From cae6c95978a852bfa27e091625e35fe4dfa1220f Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Wed, 4 Jan 2023 14:06:44 -0700 Subject: [PATCH 15/22] replacing $3 with $2 in SetBlockBuilderStatus --- database/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/database.go b/database/database.go index d966bf22..78a5a321 100644 --- a/database/database.go +++ b/database/database.go @@ -465,7 +465,7 @@ func (s *DatabaseService) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderE } func (s *DatabaseService) SetBlockBuilderStatus(pubkey string, builderStatusCode common.BuilderStatus) error { - query := `UPDATE ` + vars.TableBlockBuilder + ` SET status=$1 WHERE builder_pubkey=$3;` + query := `UPDATE ` + vars.TableBlockBuilder + ` SET status=$1 WHERE builder_pubkey=$2;` _, err := s.DB.Exec(query, uint8(builderStatusCode), pubkey) return err } From d9845af66ade1d0ab0ef4cc20c895b06dccf7611 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Thu, 5 Jan 2023 11:45:23 -0500 Subject: [PATCH 16/22] unit test initial pass --- beaconclient/mock_multi_beacon_client.go | 53 ++++++++++++++ database/database_test.go | 10 ++- datastore/redis.go | 2 +- datastore/redis_test.go | 38 ++++++++++ services/api/blocksim_ratelimiter.go | 5 ++ services/api/mock_blocksim_ratelimiter.go | 16 +++++ services/api/service.go | 4 +- services/api/service_test.go | 86 +++++++++++++++++++++++ 8 files changed, 208 insertions(+), 6 deletions(-) create mode 100644 beaconclient/mock_multi_beacon_client.go create mode 100644 services/api/mock_blocksim_ratelimiter.go diff --git a/beaconclient/mock_multi_beacon_client.go b/beaconclient/mock_multi_beacon_client.go new file mode 100644 index 00000000..95230242 --- /dev/null +++ b/beaconclient/mock_multi_beacon_client.go @@ -0,0 +1,53 @@ +package beaconclient + +import ( + "sync" + + "github.com/flashbots/go-boost-utils/types" +) + +type MockMultiBeaconClient struct { + mu sync.RWMutex +} + +func NewMockMultiBeaconClient() *MockMultiBeaconClient { + return &MockMultiBeaconClient{ + mu: sync.RWMutex{}, + } +} + +func (*MockMultiBeaconClient) SubscribeToHeadEvents(slotC chan HeadEventData) {} + +func (*MockMultiBeaconClient) BestSyncStatus() (*SyncStatusPayloadData, error) { + return &SyncStatusPayloadData{HeadSlot: 1}, nil +} + +func (*MockMultiBeaconClient) FetchValidators(headSlot uint64) (map[types.PubkeyHex]ValidatorResponseEntry, error) { + return nil, nil +} + +func (*MockMultiBeaconClient) GetProposerDuties(epoch uint64) (*ProposerDutiesResponse, error) { + return nil, nil +} + +func (*MockMultiBeaconClient) PublishBlock(block *types.SignedBeaconBlock) (code int, err error) { + return 0, nil +} + +func (*MockMultiBeaconClient) GetGenesis() (*GetGenesisResponse, error) { + resp := &GetGenesisResponse{} + resp.Data.GenesisTime = 0 + return resp, nil +} + +func (*MockMultiBeaconClient) GetSpec() (spec *GetSpecResponse, err error) { + return nil, nil +} + +func (*MockMultiBeaconClient) GetBlock(blockID string) (block *GetBlockResponse, err error) { + return nil, nil +} + +func (*MockMultiBeaconClient) GetRandao(slot uint64) (spec *GetRandaoResponse, err error) { + return nil, nil +} diff --git a/database/database_test.go b/database/database_test.go index e54ec546..2cc5b0e8 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -28,9 +28,9 @@ func createValidatorRegistration(pubKey string) ValidatorRegistrationEntry { func resetDatabase(t *testing.T) *DatabaseService { t.Helper() - if !runDBTests { - t.Skip("Skipping database tests") - } + // if !runDBTests { + // t.Skip("Skipping database tests") + // } // Wipe test database _db, err := sqlx.Connect("postgres", testDBDSN) @@ -137,3 +137,7 @@ func TestMigrations(t *testing.T) { require.NoError(t, err) require.Equal(t, len(migrations.Migrations.Migrations), rowCount) } + +func TestUpsertBuilderDemotion(t *testing.T) { + _ = resetDatabase(t) +} diff --git a/datastore/redis.go b/datastore/redis.go index b0a37edc..2d105d1a 100644 --- a/datastore/redis.go +++ b/datastore/redis.go @@ -318,7 +318,7 @@ func (r *RedisCache) GetBidTrace(slot uint64, proposerPubkey, blockHash string) } func (r *RedisCache) SetBuilderStatus(builderPubkey string, status common.BuilderStatus) (err error) { - return r.client.HSet(context.Background(), r.keyBuilderStatus, builderPubkey, status).Err() + return r.client.HSet(context.Background(), r.keyBuilderStatus, builderPubkey, uint8(status)).Err() } func (r *RedisCache) GetBuilderStatus(builderPubkey string) (status common.BuilderStatus, err error) { diff --git a/datastore/redis_test.go b/datastore/redis_test.go index bb9d11a1..786c6fd2 100644 --- a/datastore/redis_test.go +++ b/datastore/redis_test.go @@ -256,3 +256,41 @@ func TestRedisURIs(t *testing.T) { _, err = NewRedisCache(malformURL, "") require.Error(t, err) } + +func TestBuilderStatus(t *testing.T) { + builderPK := "0xdeadbeef" + cache := setupTestRedis(t) + + // Base status should be low-prio. + status, err := cache.GetBuilderStatus(builderPK) + require.NoError(t, err) + require.Equal(t, status, common.LowPrio) + + // Update to optimisitic. + err = cache.SetBuilderStatus(builderPK, common.Optimistic) + require.NoError(t, err) + + // Now status should be optimistic. + status, err = cache.GetBuilderStatus(builderPK) + require.NoError(t, err) + require.Equal(t, status, common.Optimistic) +} + +func TestBuilderCollateral(t *testing.T) { + builderPK := "0xdeadbeef" + cache := setupTestRedis(t) + + // Base collateral should be an empty string. + collateral, err := cache.GetBuilderCollateral(builderPK) + require.NoError(t, err) + require.Equal(t, collateral, "") + + // Update to be non-empty string. + err = cache.SetBuilderCollateral(builderPK, "12345") + require.NoError(t, err) + + // Now collateral should be "12345". + collateral, err = cache.GetBuilderCollateral(builderPK) + require.NoError(t, err) + require.Equal(t, collateral, "12345") +} diff --git a/services/api/blocksim_ratelimiter.go b/services/api/blocksim_ratelimiter.go index 5c47da5f..b00b42eb 100644 --- a/services/api/blocksim_ratelimiter.go +++ b/services/api/blocksim_ratelimiter.go @@ -19,6 +19,11 @@ var ( ErrSimulationFailed = errors.New("simulation failed") ) +type IBlockSimRateLimiter interface { + send(context context.Context, payload *BuilderBlockValidationRequest, isHighPrio bool) error + currentCounter() int64 +} + var maxConcurrentBlocks = int64(cli.GetEnvInt("BLOCKSIM_MAX_CONCURRENT", 4)) // 0 for no maximum type BlockSimulationRateLimiter struct { diff --git a/services/api/mock_blocksim_ratelimiter.go b/services/api/mock_blocksim_ratelimiter.go new file mode 100644 index 00000000..b9108bde --- /dev/null +++ b/services/api/mock_blocksim_ratelimiter.go @@ -0,0 +1,16 @@ +package api + +import ( + "context" +) + +type MockBlockSimulationRateLimiter struct { +} + +func (m *MockBlockSimulationRateLimiter) send(context context.Context, payload *BuilderBlockValidationRequest, isHighPrio bool) error { + return nil +} + +func (m *MockBlockSimulationRateLimiter) currentCounter() int64 { + return 0 +} diff --git a/services/api/service.go b/services/api/service.go index 2e488d8a..47688b32 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -131,7 +131,7 @@ type RelayAPI struct { proposerDutiesSlot uint64 isUpdatingProposerDuties uberatomic.Bool - blockSimRateLimiter *BlockSimulationRateLimiter + blockSimRateLimiter IBlockSimRateLimiter activeValidatorC chan types.PubkeyHex validatorRegC chan types.SignedValidatorRegistration @@ -1113,7 +1113,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque var builderCollateral types.U256Str err = builderCollateral.UnmarshalText([]byte(builderCollateralStr)) if err != nil { - log.WithError(err).Error("could parse builder collateral string") + log.WithError(err).Error("could not parse builder collateral string") builderCollateral = ZeroU256 } diff --git a/services/api/service_test.go b/services/api/service_test.go index bff3f441..3f52cf62 100644 --- a/services/api/service_test.go +++ b/services/api/service_test.go @@ -5,10 +5,12 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strconv" "testing" "time" "github.com/alicebob/miniredis/v2" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/flashbots/go-boost-utils/bls" "github.com/flashbots/go-boost-utils/types" "github.com/flashbots/mev-boost-relay/beaconclient" @@ -274,3 +276,87 @@ func TestDataApiGetDataProposerPayloadDelivered(t *testing.T) { } }) } + +func TestBuilderApiSubmitNewBlockOptimistic(t *testing.T) { + // Setup test key pair. + sk, _, err := bls.GenerateNewKeypair() + require.NoError(t, err) + blsPubkey := bls.PublicKeyFromSecretKey(sk) + var pubkey types.PublicKey + err = pubkey.FromSlice(blsPubkey.Compress()) + require.NoError(t, err) + + // Test values. + slot := uint64(42) + feeRecipient := types.Address{0x02} + collateral := 1000 + randao := "01234567890123456789012345678901" + var random types.Hash + err = random.FromSlice([]byte(randao)) + require.NoError(t, err) + var txn hexutil.Bytes + err = txn.UnmarshalText([]byte("0x03")) + require.NoError(t, err) + + // Setting up test backend. + backend := newTestBackend(t, 1) + backend.relay.expectedPrevRandao = randaoHelper{ + slot: slot, + prevRandao: random.String(), + } + backend.relay.genesisInfo = &beaconclient.GetGenesisResponse{} + backend.relay.genesisInfo.Data.GenesisTime = 0 + backend.relay.proposerDutiesMap = map[uint64]*types.RegisterValidatorRequestMessage{ + slot: &types.RegisterValidatorRequestMessage{ + FeeRecipient: feeRecipient, + GasLimit: 5000, + Timestamp: 0xffffffff, + Pubkey: types.PublicKey{}, + }, + } + backend.relay.opts.BlockBuilderAPI = true + backend.relay.beaconClient = beaconclient.NewMockMultiBeaconClient() + backend.relay.blockSimRateLimiter = &MockBlockSimulationRateLimiter{} + go backend.relay.StartServer() + time.Sleep(1 * time.Second) + + // Set up request. + path := "/relay/v1/builder/blocks" + bidTrace := &types.BidTrace{ + Slot: slot, + BuilderPubkey: pubkey, + ProposerFeeRecipient: feeRecipient, + Value: types.IntToU256(uint64(collateral) - 1), + } + signature, err := types.SignMessage(bidTrace, backend.relay.opts.EthNetDetails.DomainBuilder, sk) + require.NoError(t, err) + req := &types.BuilderSubmitBlockRequest{ + Message: bidTrace, + Signature: signature, + ExecutionPayload: &types.ExecutionPayload{ + Timestamp: slot * 12, // 12 seconds per slot. + Transactions: []hexutil.Bytes{txn}, + Random: random, + }, + } + + // Prepare redis. + err = backend.redis.SetStats(datastore.RedisStatsFieldSlotLastPayloadDelivered, req.Message.Slot-1) + require.NoError(t, err) + err = backend.redis.SetBuilderStatus(pubkey.String(), common.Optimistic) + require.NoError(t, err) + err = backend.redis.SetBuilderCollateral(pubkey.String(), strconv.Itoa(collateral)) + require.NoError(t, err) + + // Issue the request. + rr := backend.request(http.MethodPost, path, req) + require.Equal(t, http.StatusOK, rr.Code) + + // Let updates happen async. + time.Sleep(2 * time.Second) + + // Check status in redis is still optimistic. + outStatus, err := backend.redis.GetBuilderStatus(pubkey.String()) + require.NoError(t, err) + require.Equal(t, outStatus, common.Optimistic) +} From fe81a12359ad639f302746217664ef95a1ed2593 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Thu, 5 Jan 2023 11:53:04 -0500 Subject: [PATCH 17/22] remove bad test --- database/database_test.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/database/database_test.go b/database/database_test.go index 2cc5b0e8..e54ec546 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -28,9 +28,9 @@ func createValidatorRegistration(pubKey string) ValidatorRegistrationEntry { func resetDatabase(t *testing.T) *DatabaseService { t.Helper() - // if !runDBTests { - // t.Skip("Skipping database tests") - // } + if !runDBTests { + t.Skip("Skipping database tests") + } // Wipe test database _db, err := sqlx.Connect("postgres", testDBDSN) @@ -137,7 +137,3 @@ func TestMigrations(t *testing.T) { require.NoError(t, err) require.Equal(t, len(migrations.Migrations.Migrations), rowCount) } - -func TestUpsertBuilderDemotion(t *testing.T) { - _ = resetDatabase(t) -} From 8267845c45506c781323f1dafc22d7eaef578dac Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Thu, 5 Jan 2023 12:22:58 -0500 Subject: [PATCH 18/22] Justin's comments round 2 --- common/common.go | 4 +-- database/types.go | 6 ++-- datastore/redis.go | 26 ++++++++-------- services/api/service.go | 47 +++++++++++++---------------- services/housekeeper/housekeeper.go | 6 ++-- 5 files changed, 42 insertions(+), 47 deletions(-) diff --git a/common/common.go b/common/common.go index 1fc3cade..56ac139d 100644 --- a/common/common.go +++ b/common/common.go @@ -33,10 +33,10 @@ const ( func (b BuilderStatus) String() string { switch b { - case Optimistic: - return "optimistic" case HighPrio: return "high-prio" + case Optimistic: + return "optimistic" case Blacklisted: return "blacklisted" default: diff --git a/database/types.go b/database/types.go index 6348e7bc..0f1739ed 100644 --- a/database/types.go +++ b/database/types.go @@ -188,9 +188,9 @@ type BlockBuilderEntry struct { BuilderPubkey string `db:"builder_pubkey" json:"builder_pubkey"` Description string `db:"description" json:"description"` - Status uint8 `db:"status" json:"status"` - Collateral string `db:"collateral_value" json:"collateral_value"` - CollateralID uint64 `db:"collateral_id" json:"collateral_id"` + Status uint8 `db:"status" json:"status"` + CollateralValue string `db:"collateral_value" json:"collateral_value"` + CollateralID uint64 `db:"collateral_id" json:"collateral_id"` LastSubmissionID sql.NullInt64 `db:"last_submission_id" json:"last_submission_id"` LastSubmissionSlot uint64 `db:"last_submission_slot" json:"last_submission_slot"` diff --git a/datastore/redis.go b/datastore/redis.go index 2d105d1a..b643981a 100644 --- a/datastore/redis.go +++ b/datastore/redis.go @@ -69,11 +69,11 @@ type RedisCache struct { keyKnownValidators string keyValidatorRegistrationTimestamp string - keyRelayConfig string - keyStats string - keyProposerDuties string - keyBuilderStatus string - keyBuilderCollateral string + keyRelayConfig string + keyStats string + keyProposerDuties string + keyBlockBuilderStatus string + keyBlockBuilderCollateral string } func NewRedisCache(redisURI, prefix string) (*RedisCache, error) { @@ -98,10 +98,10 @@ func NewRedisCache(redisURI, prefix string) (*RedisCache, error) { keyValidatorRegistrationTimestamp: fmt.Sprintf("%s/%s:validator-registration-timestamp", redisPrefix, prefix), keyRelayConfig: fmt.Sprintf("%s/%s:relay-config", redisPrefix, prefix), - keyStats: fmt.Sprintf("%s/%s:stats", redisPrefix, prefix), - keyProposerDuties: fmt.Sprintf("%s/%s:proposer-duties", redisPrefix, prefix), - keyBuilderStatus: fmt.Sprintf("%s/%s:block-builder-status", redisPrefix, prefix), - keyBuilderCollateral: fmt.Sprintf("%s/%s:block-builder-collateral", redisPrefix, prefix), + keyStats: fmt.Sprintf("%s/%s:stats", redisPrefix, prefix), + keyProposerDuties: fmt.Sprintf("%s/%s:proposer-duties", redisPrefix, prefix), + keyBlockBuilderStatus: fmt.Sprintf("%s/%s:block-builder-status", redisPrefix, prefix), + keyBlockBuilderCollateral: fmt.Sprintf("%s/%s:block-builder-collateral", redisPrefix, prefix), }, nil } @@ -318,11 +318,11 @@ func (r *RedisCache) GetBidTrace(slot uint64, proposerPubkey, blockHash string) } func (r *RedisCache) SetBuilderStatus(builderPubkey string, status common.BuilderStatus) (err error) { - return r.client.HSet(context.Background(), r.keyBuilderStatus, builderPubkey, uint8(status)).Err() + return r.client.HSet(context.Background(), r.keyBlockBuilderStatus, builderPubkey, uint8(status)).Err() } func (r *RedisCache) GetBuilderStatus(builderPubkey string) (status common.BuilderStatus, err error) { - res, err := r.client.HGet(context.Background(), r.keyBuilderStatus, builderPubkey).Result() + res, err := r.client.HGet(context.Background(), r.keyBlockBuilderStatus, builderPubkey).Result() if errors.Is(err, redis.Nil) { return status, nil } @@ -334,11 +334,11 @@ func (r *RedisCache) GetBuilderStatus(builderPubkey string) (status common.Build } func (r *RedisCache) SetBuilderCollateral(builderPubkey, value string) (err error) { - return r.client.HSet(context.Background(), r.keyBuilderCollateral, builderPubkey, value).Err() + return r.client.HSet(context.Background(), r.keyBlockBuilderCollateral, builderPubkey, value).Err() } func (r *RedisCache) GetBuilderCollateral(builderPubkey string) (value string, err error) { - res, err := r.client.HGet(context.Background(), r.keyBuilderCollateral, builderPubkey).Result() + res, err := r.client.HGet(context.Background(), r.keyBlockBuilderCollateral, builderPubkey).Result() if errors.Is(err, redis.Nil) { return "", nil } diff --git a/services/api/service.go b/services/api/service.go index 47688b32..c7cc5883 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -416,8 +416,7 @@ func (api *RelayAPI) simulateBlock(opts blockSimOptions) error { // Demotes a set of builders who have matching collateral IDs. This function // continues to try demoting even if an error is encountered. -func (api *RelayAPI) demoteBuildersCollateralID(builderPubkey string) { - collateralPubkeys := []string{builderPubkey} +func (api *RelayAPI) demoteBuildersByCollateralID(builderPubkey string) { // Fetch builder collateral_id. builder, err := api.db.GetBlockBuilderByPubkey(builderPubkey) if err != nil { @@ -426,16 +425,14 @@ func (api *RelayAPI) demoteBuildersCollateralID(builderPubkey string) { collateralID := builder.CollateralID // Fetch additional builder pubkeys using the collateral_id. - otherBuilders, err := api.db.GetBlockBuildersFromCollateralID(collateralID) + allBuilders, err := api.db.GetBlockBuildersFromCollateralID(collateralID) if err != nil { - api.log.WithError(err).Error("unable to get extra builders from collateral id") - } - for _, b := range otherBuilders { - collateralPubkeys = append(collateralPubkeys, b.BuilderPubkey) + api.log.WithError(err).Error("unable to get builders from collateral id") } // Demote all the pubkeys in both redis and the db. - for _, pk := range collateralPubkeys { + for _, b := range allBuilders { + pk := b.BuilderPubkey err = api.redis.SetBuilderStatus(pk, common.LowPrio) if err != nil { api.log.WithError(err).Error("could not set builder status in redis") @@ -456,13 +453,13 @@ func (api *RelayAPI) startOptimisticBlockProcessor() { api.log.WithError(err).Error("block simulation failed") builderPubkey := opts.req.Message.BuilderPubkey.String() // Validation failed, demote all builders with the collateral id. - api.demoteBuildersCollateralID(builderPubkey) + api.demoteBuildersByCollateralID(builderPubkey) bidTrace := &common.BidTraceV2{ BidTrace: *opts.req.Message, } // Upsert into the builder demotion table but without the - // blinded block or the validator registration, becuase we don't + // blinded block or the validator registration, because we don't // know if this bid will be accepted. err = api.db.UpsertBuilderDemotion(bidTrace, nil, nil) if err != nil { @@ -945,7 +942,7 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) log.WithError(err).Error("failed to simulate signed block") builderPubkey := bidTrace.BuilderPubkey.String() // Validation failed, demote all builders with the collateral id. - api.demoteBuildersCollateralID(builderPubkey) + api.demoteBuildersByCollateralID(builderPubkey) registrationEntry, err := api.db.GetValidatorRegistration(bidTrace.ProposerPubkey.String()) if err != nil { @@ -1087,8 +1084,7 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque builderStatus, err := api.redis.GetBuilderStatus(payload.Message.BuilderPubkey.String()) log = log.WithFields(logrus.Fields{ - "builderStatus": builderStatus, - "builderStatusStr": builderStatus.String(), + "builderStatus": builderStatus.String(), }) if err != nil { log.WithError(err).Error("could not get block builder status") @@ -1163,12 +1159,11 @@ func (api *RelayAPI) handleSubmitNewBlock(w http.ResponseWriter, req *http.Reque } log = log.WithFields(logrus.Fields{ - "builderStatus": builderStatus, - "builderStatusStr": builderStatus.String(), - "proposerPubkey": payload.Message.ProposerPubkey.String(), - "parentHash": payload.Message.ParentHash.String(), - "value": payload.Message.Value.String(), - "tx": len(payload.ExecutionPayload.Transactions), + "builderStatus": builderStatus.String(), + "proposerPubkey": payload.Message.ProposerPubkey.String(), + "parentHash": payload.Message.ParentHash.String(), + "value": payload.Message.Value.String(), + "tx": len(payload.ExecutionPayload.Transactions), }) if payload.Message.Slot <= api.headSlot.Load() { @@ -1369,25 +1364,25 @@ func (api *RelayAPI) handleInternalBuilderStatus(w http.ResponseWriter, req *htt "isHighPrio": isHighPrio, "isBlacklisted": isBlacklisted, }).Info("updating builder status") - code := common.LowPrio + status := common.LowPrio if isBlacklisted { - code = common.Blacklisted + status = common.Blacklisted } else if isOptimistic { - code = common.Optimistic + status = common.Optimistic } else if isHighPrio { - code = common.HighPrio + status = common.HighPrio } - err := api.redis.SetBuilderStatus(builderPubkey, code) + err := api.redis.SetBuilderStatus(builderPubkey, status) if err != nil { api.log.WithError(err).Error("could not set builder status in redis") } - err = api.db.SetBlockBuilderStatus(builderPubkey, code) + err = api.db.SetBlockBuilderStatus(builderPubkey, status) if err != nil { api.log.WithError(err).Error("could not set builder status in database") } - api.RespondOK(w, struct{ newStatus string }{newStatus: code.String()}) + api.RespondOK(w, struct{ newStatus string }{newStatus: status.String()}) } } diff --git a/services/housekeeper/housekeeper.go b/services/housekeeper/housekeeper.go index f86b6151..9593d537 100644 --- a/services/housekeeper/housekeeper.go +++ b/services/housekeeper/housekeeper.go @@ -338,9 +338,9 @@ func (hk *Housekeeper) updateBlockBuildersInRedis() { hk.log.Infof("updating %d block builders in Redis...", len(builders)) for _, builder := range builders { - code := common.BuilderStatus(builder.Status) - hk.log.Infof("updating block builder in Redis: %s - %s", builder.BuilderPubkey, code.String()) - err = hk.redis.SetBuilderStatus(builder.BuilderPubkey, code) + status := common.BuilderStatus(builder.Status) + hk.log.Infof("updating block builder in Redis: %s - %s", builder.BuilderPubkey, status.String()) + err = hk.redis.SetBuilderStatus(builder.BuilderPubkey, status) if err != nil { hk.log.WithError(err).Error("failed to set builder status in redis") } From ac4aaa2c8b741ae9987f10a15ce93468442af811 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Fri, 6 Jan 2023 20:14:23 -0500 Subject: [PATCH 19/22] Unit testing block simulation path --- database/mockdb.go | 25 ++- services/api/mock_blocksim_ratelimiter.go | 3 +- services/api/optimistic_test.go | 213 ++++++++++++++++++++++ services/api/service_test.go | 86 --------- 4 files changed, 237 insertions(+), 90 deletions(-) create mode 100644 services/api/optimistic_test.go diff --git a/database/mockdb.go b/database/mockdb.go index 798fe9bf..cae5958a 100644 --- a/database/mockdb.go +++ b/database/mockdb.go @@ -7,7 +7,11 @@ import ( "github.com/flashbots/mev-boost-relay/common" ) -type MockDB struct{} +type MockDB struct { + Builders map[string]*BlockBuilderEntry + Demotions map[string]bool + Refunds map[string]bool +} func (db MockDB) NumRegisteredValidators() (count uint64, err error) { return 0, nil @@ -86,10 +90,12 @@ func (db MockDB) GetBlockBuilders() ([]*BlockBuilderEntry, error) { } func (db MockDB) GetBlockBuilderByPubkey(pubkey string) (*BlockBuilderEntry, error) { - return nil, nil + return db.Builders[pubkey], nil } func (db MockDB) SetBlockBuilderStatus(pubkey string, builderStatus common.BuilderStatus) error { + builder := db.Builders[pubkey] + builder.Status = uint8(builderStatus) return nil } @@ -102,9 +108,22 @@ func (db MockDB) IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error } func (db MockDB) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { + pk := bidTrace.BuilderPubkey.String() + db.Demotions[pk] = true + + // Refundable case. + if signedBlindedBeaconBlock != nil && signedValidatorRegistration != nil { + db.Refunds[pk] = true + } return nil } func (db MockDB) GetBlockBuildersFromCollateralID(collateralID uint64) ([]*BlockBuilderEntry, error) { - return nil, nil + res := []*BlockBuilderEntry{} + for _, v := range db.Builders { + if v.CollateralID == collateralID { + res = append(res, v) + } + } + return res, nil } diff --git a/services/api/mock_blocksim_ratelimiter.go b/services/api/mock_blocksim_ratelimiter.go index b9108bde..65bde20f 100644 --- a/services/api/mock_blocksim_ratelimiter.go +++ b/services/api/mock_blocksim_ratelimiter.go @@ -5,10 +5,11 @@ import ( ) type MockBlockSimulationRateLimiter struct { + simulationError error } func (m *MockBlockSimulationRateLimiter) send(context context.Context, payload *BuilderBlockValidationRequest, isHighPrio bool) error { - return nil + return m.simulationError } func (m *MockBlockSimulationRateLimiter) currentCounter() int64 { diff --git a/services/api/optimistic_test.go b/services/api/optimistic_test.go new file mode 100644 index 00000000..3b2c63b1 --- /dev/null +++ b/services/api/optimistic_test.go @@ -0,0 +1,213 @@ +package api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/flashbots/go-boost-utils/bls" + "github.com/flashbots/go-boost-utils/types" + "github.com/flashbots/mev-boost-relay/beaconclient" + "github.com/flashbots/mev-boost-relay/common" + "github.com/flashbots/mev-boost-relay/database" + "github.com/flashbots/mev-boost-relay/datastore" + "github.com/stretchr/testify/require" + blst "github.com/supranational/blst/bindings/go" +) + +const ( + slot = uint64(42) + collateral = 1000 + collateralID = 567 + randao = "01234567890123456789012345678901" + path = "/relay/v1/builder/blocks" +) + +var ( + feeRecipient = types.Address{0x02} +) + +type optimisticTestOpts struct { + pubkey types.PublicKey + secretkey *blst.SecretKey + simulationErr error + blockValue types.U256Str +} + +func getTestRandomHash(t *testing.T) types.Hash { + var random types.Hash + err := random.FromSlice([]byte(randao)) + require.NoError(t, err) + return random +} + +func startTestBackend(t *testing.T) (types.PublicKey, *blst.SecretKey, *testBackend) { + // Setup test key pair. + sk, _, err := bls.GenerateNewKeypair() + require.NoError(t, err) + blsPubkey := bls.PublicKeyFromSecretKey(sk) + var pubkey types.PublicKey + err = pubkey.FromSlice(blsPubkey.Compress()) + require.NoError(t, err) + pkStr := pubkey.String() + + // Setup test backend. + backend := newTestBackend(t, 1) + backend.relay.expectedPrevRandao = randaoHelper{ + slot: slot, + prevRandao: getTestRandomHash(t).String(), + } + backend.relay.genesisInfo = &beaconclient.GetGenesisResponse{} + backend.relay.genesisInfo.Data.GenesisTime = 0 + backend.relay.proposerDutiesMap = map[uint64]*types.RegisterValidatorRequestMessage{ + slot: &types.RegisterValidatorRequestMessage{ + FeeRecipient: feeRecipient, + GasLimit: 5000, + Timestamp: 0xffffffff, + Pubkey: types.PublicKey{}, + }, + } + backend.relay.opts.BlockBuilderAPI = true + backend.relay.beaconClient = beaconclient.NewMockMultiBeaconClient() + backend.relay.blockSimRateLimiter = &MockBlockSimulationRateLimiter{} + backend.relay.db = &database.MockDB{ + Builders: map[string]*database.BlockBuilderEntry{ + pkStr: &database.BlockBuilderEntry{ + BuilderPubkey: pkStr, + CollateralID: collateralID, + }, + }, + Demotions: map[string]bool{}, + Refunds: map[string]bool{}, + } + go backend.relay.StartServer() + time.Sleep(1 * time.Second) + + // Prepare redis. + err = backend.redis.SetStats(datastore.RedisStatsFieldSlotLastPayloadDelivered, slot-1) + require.NoError(t, err) + err = backend.redis.SetBuilderStatus(pkStr, common.Optimistic) + require.NoError(t, err) + err = backend.redis.SetBuilderCollateral(pkStr, strconv.Itoa(collateral)) + require.NoError(t, err) + + // Prepare db. + err = backend.relay.db.SetBlockBuilderStatus(pkStr, common.Optimistic) + require.NoError(t, err) + + return pubkey, sk, backend +} + +func runOptimisticBlockSubmission(t *testing.T, opts optimisticTestOpts, backend *testBackend) *httptest.ResponseRecorder { + var txn hexutil.Bytes + err := txn.UnmarshalText([]byte("0x03")) + require.NoError(t, err) + + backend.relay.blockSimRateLimiter = &MockBlockSimulationRateLimiter{ + simulationError: opts.simulationErr, + } + + // Set up request. + bidTrace := &types.BidTrace{ + Slot: slot, + BuilderPubkey: opts.pubkey, + ProposerFeeRecipient: feeRecipient, + Value: opts.blockValue, + } + signature, err := types.SignMessage(bidTrace, backend.relay.opts.EthNetDetails.DomainBuilder, opts.secretkey) + require.NoError(t, err) + req := &types.BuilderSubmitBlockRequest{ + Message: bidTrace, + Signature: signature, + ExecutionPayload: &types.ExecutionPayload{ + Timestamp: slot * 12, // 12 seconds per slot. + Transactions: []hexutil.Bytes{txn}, + Random: getTestRandomHash(t), + }, + } + + rr := backend.request(http.MethodPost, path, req) + + // Let updates happen async. + time.Sleep(2 * time.Second) + + return rr +} + +func TestBuilderApiSubmitNewBlockOptimistic(t *testing.T) { + testCases := []struct { + description string + wantStatus common.BuilderStatus + simulationErr error + expectDemotion bool + httpCode uint64 + blockValue types.U256Str + }{ + { + description: "success_value_less_than_collateral", + wantStatus: common.Optimistic, + simulationErr: nil, + expectDemotion: false, + httpCode: 200, // success + blockValue: types.IntToU256(uint64(collateral) - 1), + }, + { + description: "success_value_greater_than_collateral", + wantStatus: common.Optimistic, + simulationErr: nil, + expectDemotion: false, + httpCode: 200, // success + blockValue: types.IntToU256(uint64(collateral) + 1), + }, + { + description: "failure_value_less_than_collateral", + wantStatus: common.LowPrio, + simulationErr: fmt.Errorf("fake error"), + expectDemotion: true, + httpCode: 200, // success (in optimistic mode, block sim failure will happen async) + blockValue: types.IntToU256(uint64(collateral) - 1), + }, + { + description: "failure_value_more_than_collateral", + wantStatus: common.Optimistic, + simulationErr: fmt.Errorf("fake error"), + expectDemotion: false, + httpCode: 400, // failure (in pessimistic mode, block sim failure happens in response path) + blockValue: types.IntToU256(uint64(collateral) + 1), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + pk, sk, backend := startTestBackend(t) + pkStr := pk.String() + rr := runOptimisticBlockSubmission(t, optimisticTestOpts{ + secretkey: sk, + pubkey: pk, + simulationErr: tc.simulationErr, + blockValue: tc.blockValue, + }, backend) + + // Check http code. + require.Equal(t, uint64(rr.Code), tc.httpCode) + + // Check status in redis. + outStatus, err := backend.redis.GetBuilderStatus(pkStr) + require.NoError(t, err) + require.Equal(t, outStatus, tc.wantStatus) + + // Check status in db. + dbBuilder, err := backend.relay.db.GetBlockBuilderByPubkey(pkStr) + require.NoError(t, err) + require.Equal(t, common.BuilderStatus(dbBuilder.Status), tc.wantStatus) + + // Check demotion status is set to true. + mockDB := backend.relay.db.(*database.MockDB) + require.Equal(t, mockDB.Demotions[pkStr], tc.expectDemotion) + }) + } +} diff --git a/services/api/service_test.go b/services/api/service_test.go index 3f52cf62..bff3f441 100644 --- a/services/api/service_test.go +++ b/services/api/service_test.go @@ -5,12 +5,10 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "strconv" "testing" "time" "github.com/alicebob/miniredis/v2" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/flashbots/go-boost-utils/bls" "github.com/flashbots/go-boost-utils/types" "github.com/flashbots/mev-boost-relay/beaconclient" @@ -276,87 +274,3 @@ func TestDataApiGetDataProposerPayloadDelivered(t *testing.T) { } }) } - -func TestBuilderApiSubmitNewBlockOptimistic(t *testing.T) { - // Setup test key pair. - sk, _, err := bls.GenerateNewKeypair() - require.NoError(t, err) - blsPubkey := bls.PublicKeyFromSecretKey(sk) - var pubkey types.PublicKey - err = pubkey.FromSlice(blsPubkey.Compress()) - require.NoError(t, err) - - // Test values. - slot := uint64(42) - feeRecipient := types.Address{0x02} - collateral := 1000 - randao := "01234567890123456789012345678901" - var random types.Hash - err = random.FromSlice([]byte(randao)) - require.NoError(t, err) - var txn hexutil.Bytes - err = txn.UnmarshalText([]byte("0x03")) - require.NoError(t, err) - - // Setting up test backend. - backend := newTestBackend(t, 1) - backend.relay.expectedPrevRandao = randaoHelper{ - slot: slot, - prevRandao: random.String(), - } - backend.relay.genesisInfo = &beaconclient.GetGenesisResponse{} - backend.relay.genesisInfo.Data.GenesisTime = 0 - backend.relay.proposerDutiesMap = map[uint64]*types.RegisterValidatorRequestMessage{ - slot: &types.RegisterValidatorRequestMessage{ - FeeRecipient: feeRecipient, - GasLimit: 5000, - Timestamp: 0xffffffff, - Pubkey: types.PublicKey{}, - }, - } - backend.relay.opts.BlockBuilderAPI = true - backend.relay.beaconClient = beaconclient.NewMockMultiBeaconClient() - backend.relay.blockSimRateLimiter = &MockBlockSimulationRateLimiter{} - go backend.relay.StartServer() - time.Sleep(1 * time.Second) - - // Set up request. - path := "/relay/v1/builder/blocks" - bidTrace := &types.BidTrace{ - Slot: slot, - BuilderPubkey: pubkey, - ProposerFeeRecipient: feeRecipient, - Value: types.IntToU256(uint64(collateral) - 1), - } - signature, err := types.SignMessage(bidTrace, backend.relay.opts.EthNetDetails.DomainBuilder, sk) - require.NoError(t, err) - req := &types.BuilderSubmitBlockRequest{ - Message: bidTrace, - Signature: signature, - ExecutionPayload: &types.ExecutionPayload{ - Timestamp: slot * 12, // 12 seconds per slot. - Transactions: []hexutil.Bytes{txn}, - Random: random, - }, - } - - // Prepare redis. - err = backend.redis.SetStats(datastore.RedisStatsFieldSlotLastPayloadDelivered, req.Message.Slot-1) - require.NoError(t, err) - err = backend.redis.SetBuilderStatus(pubkey.String(), common.Optimistic) - require.NoError(t, err) - err = backend.redis.SetBuilderCollateral(pubkey.String(), strconv.Itoa(collateral)) - require.NoError(t, err) - - // Issue the request. - rr := backend.request(http.MethodPost, path, req) - require.Equal(t, http.StatusOK, rr.Code) - - // Let updates happen async. - time.Sleep(2 * time.Second) - - // Check status in redis is still optimistic. - outStatus, err := backend.redis.GetBuilderStatus(pubkey.String()) - require.NoError(t, err) - require.Equal(t, outStatus, common.Optimistic) -} From def09c6023a9033dd04a8a532f38085923be24a2 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Sat, 7 Jan 2023 20:18:10 -0500 Subject: [PATCH 20/22] unit test get payload optimistic --- services/api/optimistic_test.go | 129 ++++++++++++++++++++++++++++++-- services/api/service.go | 11 ++- 2 files changed, 131 insertions(+), 9 deletions(-) diff --git a/services/api/optimistic_test.go b/services/api/optimistic_test.go index 3b2c63b1..d29a52a6 100644 --- a/services/api/optimistic_test.go +++ b/services/api/optimistic_test.go @@ -24,11 +24,12 @@ const ( collateral = 1000 collateralID = 567 randao = "01234567890123456789012345678901" - path = "/relay/v1/builder/blocks" + proposerInd = uint64(987) ) var ( feeRecipient = types.Address{0x02} + errFake = fmt.Errorf("foo error") ) type optimisticTestOpts struct { @@ -45,6 +46,13 @@ func getTestRandomHash(t *testing.T) types.Hash { return random } +func getTestBlockHash(t *testing.T) types.Hash { + var blockHash types.Hash + err := blockHash.FromSlice([]byte("98765432109876543210987654321098")) + require.NoError(t, err) + return blockHash +} + func startTestBackend(t *testing.T) (types.PublicKey, *blst.SecretKey, *testBackend) { // Setup test key pair. sk, _, err := bls.GenerateNewKeypair() @@ -94,11 +102,38 @@ func startTestBackend(t *testing.T) (types.PublicKey, *blst.SecretKey, *testBack require.NoError(t, err) err = backend.redis.SetBuilderCollateral(pkStr, strconv.Itoa(collateral)) require.NoError(t, err) + err = backend.redis.SetKnownValidator(pubkey.PubkeyHex(), proposerInd) + require.NoError(t, err) + err = backend.redis.SaveExecutionPayload( + slot, + pkStr, + getTestBlockHash(t).String(), + &types.GetPayloadResponse{ + Data: &types.ExecutionPayload{ + Transactions: []hexutil.Bytes{}, + }, + }, + ) + require.NoError(t, err) + err = backend.redis.SaveBidTrace(&common.BidTraceV2{ + BidTrace: types.BidTrace{ + Slot: slot, + ProposerPubkey: pubkey, + BlockHash: getTestBlockHash(t), + BuilderPubkey: pubkey, + }, + }) + require.NoError(t, err) // Prepare db. err = backend.relay.db.SetBlockBuilderStatus(pkStr, common.Optimistic) require.NoError(t, err) + // Prepare datastore. + count, err := backend.relay.datastore.RefreshKnownValidators() + require.NoError(t, err) + require.Equal(t, count, 1) + return pubkey, sk, backend } @@ -130,7 +165,7 @@ func runOptimisticBlockSubmission(t *testing.T, opts optimisticTestOpts, backend }, } - rr := backend.request(http.MethodPost, path, req) + rr := backend.request(http.MethodPost, pathSubmitNewBlock, req) // Let updates happen async. time.Sleep(2 * time.Second) @@ -138,6 +173,41 @@ func runOptimisticBlockSubmission(t *testing.T, opts optimisticTestOpts, backend return rr } +func runOptimisticGetPayload(t *testing.T, opts optimisticTestOpts, backend *testBackend) { + var txn hexutil.Bytes + err := txn.UnmarshalText([]byte("0x03")) + require.NoError(t, err) + + backend.relay.blockSimRateLimiter = &MockBlockSimulationRateLimiter{ + simulationError: opts.simulationErr, + } + + block := &types.BlindedBeaconBlock{ + Slot: slot, + ProposerIndex: proposerInd, + Body: &types.BlindedBeaconBlockBody{ + ExecutionPayloadHeader: &types.ExecutionPayloadHeader{ + BlockHash: getTestBlockHash(t), + BlockNumber: 1234, + }, + Eth1Data: &types.Eth1Data{}, + SyncAggregate: &types.SyncAggregate{}, + }, + } + signature, err := types.SignMessage(block, backend.relay.opts.EthNetDetails.DomainBeaconProposer, opts.secretkey) + require.NoError(t, err) + req := &types.SignedBlindedBeaconBlock{ + Message: block, + Signature: signature, + } + + rr := backend.request(http.MethodPost, pathGetPayload, req) + require.Equal(t, rr.Code, http.StatusOK) + + // Let updates happen async. + time.Sleep(2 * time.Second) +} + func TestBuilderApiSubmitNewBlockOptimistic(t *testing.T) { testCases := []struct { description string @@ -166,7 +236,7 @@ func TestBuilderApiSubmitNewBlockOptimistic(t *testing.T) { { description: "failure_value_less_than_collateral", wantStatus: common.LowPrio, - simulationErr: fmt.Errorf("fake error"), + simulationErr: errFake, expectDemotion: true, httpCode: 200, // success (in optimistic mode, block sim failure will happen async) blockValue: types.IntToU256(uint64(collateral) - 1), @@ -174,7 +244,7 @@ func TestBuilderApiSubmitNewBlockOptimistic(t *testing.T) { { description: "failure_value_more_than_collateral", wantStatus: common.Optimistic, - simulationErr: fmt.Errorf("fake error"), + simulationErr: errFake, expectDemotion: false, httpCode: 400, // failure (in pessimistic mode, block sim failure happens in response path) blockValue: types.IntToU256(uint64(collateral) + 1), @@ -205,9 +275,58 @@ func TestBuilderApiSubmitNewBlockOptimistic(t *testing.T) { require.NoError(t, err) require.Equal(t, common.BuilderStatus(dbBuilder.Status), tc.wantStatus) - // Check demotion status is set to true. + // Check demotion status is set to expected. mockDB := backend.relay.db.(*database.MockDB) require.Equal(t, mockDB.Demotions[pkStr], tc.expectDemotion) }) } } + +func TestProposerApiGetPayloadOptimistic(t *testing.T) { + testCases := []struct { + description string + wantStatus common.BuilderStatus + simulationErr error + expectRefund bool + }{ + { + description: "success", + wantStatus: common.Optimistic, + simulationErr: nil, + expectRefund: false, + }, + { + description: "sim_error_refund", + wantStatus: common.LowPrio, + simulationErr: errFake, + expectRefund: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + pk, sk, backend := startTestBackend(t) + pkStr := pk.String() + runOptimisticGetPayload(t, optimisticTestOpts{ + secretkey: sk, + pubkey: pk, + simulationErr: tc.simulationErr, + }, backend) + + // Check status in redis. + outStatus, err := backend.redis.GetBuilderStatus(pkStr) + require.NoError(t, err) + require.Equal(t, outStatus, tc.wantStatus) + + // Check status in db. + dbBuilder, err := backend.relay.db.GetBlockBuilderByPubkey(pkStr) + require.NoError(t, err) + require.Equal(t, common.BuilderStatus(dbBuilder.Status), tc.wantStatus) + + // Check demotion and refund statuses are set to expected. + mockDB := backend.relay.db.(*database.MockDB) + require.Equal(t, mockDB.Demotions[pkStr], tc.expectRefund) + require.Equal(t, mockDB.Refunds[pkStr], tc.expectRefund) + }) + } +} diff --git a/services/api/service.go b/services/api/service.go index c7cc5883..2f91f3a8 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -944,6 +944,7 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) // Validation failed, demote all builders with the collateral id. api.demoteBuildersByCollateralID(builderPubkey) + signedRegistration := &types.SignedValidatorRegistration{} registrationEntry, err := api.db.GetValidatorRegistration(bidTrace.ProposerPubkey.String()) if err != nil { if errors.Is(err, sql.ErrNoRows) { @@ -952,11 +953,13 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) api.log.WithError(err).Error("error getting validator registration") } } - - signedRegistration, err := registrationEntry.ToSignedValidatorRegistration() - if err != nil { - api.log.WithError(err).Error("error converting registration entry to signed validator registration") + if registrationEntry != nil { + signedRegistration, err = registrationEntry.ToSignedValidatorRegistration() + if err != nil { + api.log.WithError(err).Error("error converting registration entry to signed validator registration") + } } + err = api.db.UpsertBuilderDemotion(bidTrace, payload, signedRegistration) if err != nil { log.WithError(err).WithFields(logrus.Fields{ From 6692741804501afd1b2fb522bb0ca9ce1e88c08c Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Sat, 7 Jan 2023 20:53:06 -0500 Subject: [PATCH 21/22] save full request from builder to justify demotion --- database/database.go | 21 +++++++++++---------- database/migrations/001_init_database.go | 2 +- database/mockdb.go | 4 ++-- database/types.go | 2 +- services/api/service.go | 18 ++++++++---------- 5 files changed, 23 insertions(+), 24 deletions(-) diff --git a/database/database.go b/database/database.go index 78a5a321..9549d902 100644 --- a/database/database.go +++ b/database/database.go @@ -46,7 +46,7 @@ type IDatabaseService interface { IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error GetBlockBuildersFromCollateralID(collateralID uint64) ([]*BlockBuilderEntry, error) - UpsertBuilderDemotion(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error + UpsertBuilderDemotion(submitBlockRequest *types.BuilderSubmitBlockRequest, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error } type DatabaseService struct { @@ -490,17 +490,18 @@ func (s *DatabaseService) DeleteExecutionPayloads(idFirst, idLast uint64) error return err } -func (s *DatabaseService) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { - if bidTrace == nil { - return fmt.Errorf("nil bidTrace invalid for UpsertBuilderDemotion") +func (s *DatabaseService) UpsertBuilderDemotion(submitBlockRequest *types.BuilderSubmitBlockRequest, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { + if submitBlockRequest == nil { + return fmt.Errorf("nil submitBlockRequest invalid for UpsertBuilderDemotion") } - _bidTrace, err := json.Marshal(bidTrace) + _submitBlockRequest, err := json.Marshal(submitBlockRequest) if err != nil { return err } + bidTrace := submitBlockRequest.Message builderDemotionEntry := BuilderDemotionEntry{ - UnsignedBidTrace: NewNullString(string(_bidTrace)), + SubmitBlockRequest: NewNullString(string(_submitBlockRequest)), Slot: bidTrace.Slot, Epoch: bidTrace.Slot / uint64(common.SlotsPerEpoch), @@ -535,8 +536,8 @@ func (s *DatabaseService) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, sig // If block_hash conflicts and we have a published block, fill in fields needed for the refund. if signedBlindedBeaconBlock != nil && signedValidatorRegistration != nil { query = `INSERT INTO ` + vars.TableBuilderDemotions + ` - (unsigned_bid_trace, signed_blinded_beacon_block, signed_validator_registration, slot, epoch, builder_pubkey, proposer_pubkey, value, fee_recipient, gas_limit, block_hash) VALUES - (:unsigned_bid_trace, :signed_blinded_beacon_block, :signed_validator_registration, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value, :fee_recipient, :gas_limit, :block_hash) + (submit_block_request, signed_blinded_beacon_block, signed_validator_registration, slot, epoch, builder_pubkey, proposer_pubkey, value, fee_recipient, gas_limit, block_hash) VALUES + (:submit_block_request, :signed_blinded_beacon_block, :signed_validator_registration, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value, :fee_recipient, :gas_limit, :block_hash) ON CONFLICT (block_hash) DO UPDATE SET signed_blinded_beacon_block = :signed_blinded_beacon_block, signed_validator_registration = :signed_validator_registration, @@ -546,8 +547,8 @@ func (s *DatabaseService) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, sig } else { // If the block_hash conflicts, then all the relevant data must be there already. query = `INSERT INTO ` + vars.TableBuilderDemotions + ` - (unsigned_bid_trace, signed_blinded_beacon_block, signed_validator_registration, slot, epoch, builder_pubkey, proposer_pubkey, value, fee_recipient, gas_limit, block_hash) VALUES - (:unsigned_bid_trace, :signed_blinded_beacon_block, :signed_validator_registration, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value, :fee_recipient, :gas_limit, :block_hash) + (submit_block_request, signed_blinded_beacon_block, signed_validator_registration, slot, epoch, builder_pubkey, proposer_pubkey, value, fee_recipient, gas_limit, block_hash) VALUES + (:submit_block_request, :signed_blinded_beacon_block, :signed_validator_registration, :slot, :epoch, :builder_pubkey, :proposer_pubkey, :value, :fee_recipient, :gas_limit, :block_hash) ON CONFLICT (block_hash) DO NOTHING` } _, err = s.DB.NamedExec(query, builderDemotionEntry) diff --git a/database/migrations/001_init_database.go b/database/migrations/001_init_database.go index 91f790d8..af3ee2c0 100644 --- a/database/migrations/001_init_database.go +++ b/database/migrations/001_init_database.go @@ -140,7 +140,7 @@ var Migration001InitDatabase = &migrate.Migration{ id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, inserted_at timestamp NOT NULL default current_timestamp, - unsigned_bid_trace json, + submit_block_request json, signed_blinded_beacon_block json, signed_validator_registration json, diff --git a/database/mockdb.go b/database/mockdb.go index cae5958a..c75ac865 100644 --- a/database/mockdb.go +++ b/database/mockdb.go @@ -107,8 +107,8 @@ func (db MockDB) IncBlockBuilderStatsAfterGetPayload(builderPubkey string) error return nil } -func (db MockDB) UpsertBuilderDemotion(bidTrace *common.BidTraceV2, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { - pk := bidTrace.BuilderPubkey.String() +func (db MockDB) UpsertBuilderDemotion(submitBlockRequest *types.BuilderSubmitBlockRequest, signedBlindedBeaconBlock *types.SignedBlindedBeaconBlock, signedValidatorRegistration *types.SignedValidatorRegistration) error { + pk := submitBlockRequest.Message.BuilderPubkey.String() db.Demotions[pk] = true // Refundable case. diff --git a/database/types.go b/database/types.go index 0f1739ed..da296854 100644 --- a/database/types.go +++ b/database/types.go @@ -205,7 +205,7 @@ type BuilderDemotionEntry struct { ID int64 `db:"id"` InsertedAt time.Time `db:"inserted_at"` - UnsignedBidTrace sql.NullString `db:"unsigned_bid_trace"` + SubmitBlockRequest sql.NullString `db:"submit_block_request"` SignedBlindedBeaconBlock sql.NullString `db:"signed_blinded_beacon_block"` SignedValidatorRegistration sql.NullString `db:"signed_validator_registration"` diff --git a/services/api/service.go b/services/api/service.go index 2f91f3a8..3e53f79c 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -455,13 +455,10 @@ func (api *RelayAPI) startOptimisticBlockProcessor() { // Validation failed, demote all builders with the collateral id. api.demoteBuildersByCollateralID(builderPubkey) - bidTrace := &common.BidTraceV2{ - BidTrace: *opts.req.Message, - } // Upsert into the builder demotion table but without the // blinded block or the validator registration, because we don't // know if this bid will be accepted. - err = api.db.UpsertBuilderDemotion(bidTrace, nil, nil) + err = api.db.UpsertBuilderDemotion(&opts.req.BuilderSubmitBlockRequest, nil, nil) if err != nil { api.log.WithError(err).Error("could not upsert bid trace") } @@ -926,16 +923,17 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) // Check if the block was valid in the optimistic case. if builderStatus == common.Optimistic { + submitBlockReq := types.BuilderSubmitBlockRequest{ + Signature: payload.Signature, + Message: &bidTrace.BidTrace, + ExecutionPayload: getPayloadResp.Data, + } simErr := api.simulateBlock(blockSimOptions{ ctx: req.Context(), log: log, highPrio: true, // manually set to true for these blocks. req: &BuilderBlockValidationRequest{ - BuilderSubmitBlockRequest: types.BuilderSubmitBlockRequest{ - Signature: payload.Signature, - Message: &bidTrace.BidTrace, - ExecutionPayload: getPayloadResp.Data, - }, + BuilderSubmitBlockRequest: submitBlockReq, }, }) if simErr != nil { @@ -960,7 +958,7 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) } } - err = api.db.UpsertBuilderDemotion(bidTrace, payload, signedRegistration) + err = api.db.UpsertBuilderDemotion(&submitBlockReq, payload, signedRegistration) if err != nil { log.WithError(err).WithFields(logrus.Fields{ "bidTrace": bidTrace, From a2c96626377ad726eb25ef3b76143b53fa6198d6 Mon Sep 17 00:00:00 2001 From: Michael Neuder Date: Wed, 11 Jan 2023 20:56:54 -0500 Subject: [PATCH 22/22] temporarily set optimistic builders as high-prio if the winning block is being simulated --- services/api/service.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/services/api/service.go b/services/api/service.go index 3e53f79c..8ec94ce5 100644 --- a/services/api/service.go +++ b/services/api/service.go @@ -331,8 +331,6 @@ func (api *RelayAPI) StartServer() (err error) { for { headEvent := <-c api.processNewSlot(headEvent.Slot) - // TODO(mikeneuder): Add checks to make sure we haven't - // optimistically built on top of a non-simulated slot. } }() @@ -923,6 +921,12 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) // Check if the block was valid in the optimistic case. if builderStatus == common.Optimistic { + // Set to high-prio while we process the winning block. + err = api.redis.SetBuilderStatus(bidTrace.BuilderPubkey.String(), common.HighPrio) + if err != nil { + log.WithError(err).Error("failed to set builder builder status") + } + submitBlockReq := types.BuilderSubmitBlockRequest{ Signature: payload.Signature, Message: &bidTrace.BidTrace, @@ -966,6 +970,12 @@ func (api *RelayAPI) handleGetPayload(w http.ResponseWriter, req *http.Request) "signedValidatorRegistration": signedRegistration, }).Error("failed to save validator refund to database") } + } else { + // Set back to optimistic because the simulation was successful. + err = api.redis.SetBuilderStatus(bidTrace.BuilderPubkey.String(), common.Optimistic) + if err != nil { + log.WithError(err).Error("failed to set builder builder status") + } } } }()