Skip to content

Commit b2af175

Browse files
committed
Fix supervisor unstuck vaults
added autobalancer callback to find potentially stuck vaults
1 parent a89895a commit b2af175

4 files changed

Lines changed: 212 additions & 37 deletions

File tree

cadence/contracts/FlowYieldVaultsAutoBalancers.cdc

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,16 @@ access(all) contract FlowYieldVaultsAutoBalancers {
2929
/// The path prefix used for StoragePath & PublicPath derivations
3030
access(all) let pathPrefix: String
3131

32+
/// Storage path for the shared execution callback that reports to the registry (one per account)
33+
access(self) let registryReportCallbackStoragePath: StoragePath
34+
35+
/// Callback resource invoked by each AutoBalancer after execution; calls Registry.reportExecutionFromCallback with its id
36+
access(all) resource RegistryReportCallback: DeFiActions.AutoBalancerExecutionCallback {
37+
access(all) fun onExecuted(balancerUUID: UInt64) {
38+
FlowYieldVaultsSchedulerRegistry.reportExecution(yieldVaultID: balancerUUID)
39+
}
40+
}
41+
3242
/* --- PUBLIC METHODS --- */
3343

3444
/// Returns the path (StoragePath or PublicPath) at which an AutoBalancer is stored with the associated
@@ -55,7 +65,7 @@ access(all) contract FlowYieldVaultsAutoBalancers {
5565
if autoBalancer == nil {
5666
return false
5767
}
58-
68+
5969
let txnIDs = autoBalancer!.getScheduledTransactionIDs()
6070
for txnID in txnIDs {
6171
if autoBalancer!.borrowScheduledTransaction(id: txnID)?.status() == FlowTransactionScheduler.Status.Scheduled {
@@ -79,24 +89,24 @@ access(all) contract FlowYieldVaultsAutoBalancers {
7989
if autoBalancer == nil {
8090
return false
8191
}
82-
92+
8393
// Check if yield vault has recurring config (should be executing periodically)
8494
let config = autoBalancer!.getRecurringConfig()
8595
if config == nil {
8696
return false // Not configured for recurring, can't be "stuck"
8797
}
88-
98+
8999
// Check if there's an active schedule
90100
if self.hasActiveSchedule(id: id) {
91101
return false // Has active schedule, not stuck
92102
}
93-
103+
94104
// Check if yield vault is overdue
95105
let nextExpected = autoBalancer!.calculateNextExecutionTimestampAsConfigured()
96106
if nextExpected == nil {
97107
return true // Can't calculate next time, likely stuck
98108
}
99-
109+
100110
// If next expected time has passed and no active schedule, yield vault is stuck
101111
return nextExpected! < getCurrentBlock().timestamp
102112
}
@@ -136,6 +146,8 @@ access(all) contract FlowYieldVaultsAutoBalancers {
136146
assert(!publishedCap,
137147
message: "Published Capability collision found when publishing AutoBalancer for UniqueIdentifier.id \(uniqueID.id) at path \(publicPath)")
138148

149+
let reportCap = self.account.capabilities.storage.issue<&{DeFiActions.AutoBalancerExecutionCallback}>(self.registryReportCallbackStoragePath)
150+
139151
// create & save AutoBalancer with optional recurring config
140152
let autoBalancer <- DeFiActions.createAutoBalancer(
141153
oracle: oracle,
@@ -147,6 +159,7 @@ access(all) contract FlowYieldVaultsAutoBalancers {
147159
recurringConfig: recurringConfig,
148160
uniqueID: uniqueID
149161
)
162+
autoBalancer.setExecutionCallback(reportCap)
150163
self.account.storage.save(<-autoBalancer, to: storagePath)
151164
let autoBalancerRef = self._borrowAutoBalancer(uniqueID.id)
152165

@@ -210,7 +223,7 @@ access(all) contract FlowYieldVaultsAutoBalancers {
210223
let publicPath = self.deriveAutoBalancerPath(id: id, storage: false) as! PublicPath
211224
// unpublish the public AutoBalancer Capability
212225
let _ = self.account.capabilities.unpublish(publicPath)
213-
226+
214227
// Collect controller IDs first (can't modify during iteration)
215228
var controllersToDelete: [UInt64] = []
216229
self.account.capabilities.storage.forEachController(forPath: storagePath, fun(_ controller: &StorageCapabilityController): Bool {
@@ -223,13 +236,24 @@ access(all) contract FlowYieldVaultsAutoBalancers {
223236
controller.delete()
224237
}
225238
}
226-
239+
227240
// load & burn the AutoBalancer (this also handles any pending scheduled transactions via burnCallback)
228241
let autoBalancer <-self.account.storage.load<@DeFiActions.AutoBalancer>(from: storagePath)
229242
Burner.burn(<-autoBalancer)
230243
}
231244

245+
access(self) fun createRegistryReportCallbackImpl(): @RegistryReportCallback {
246+
return <-create RegistryReportCallback()
247+
}
248+
232249
init() {
233250
self.pathPrefix = "FlowYieldVaultsAutoBalancer_"
251+
self.registryReportCallbackStoragePath = StoragePath(identifier: "FlowYieldVaultsRegistryReportCallback")!
252+
253+
// Ensure shared execution callback exists (reports this account's executions to Registry)
254+
if self.account.storage.type(at: self.registryReportCallbackStoragePath) == nil {
255+
self.account.storage.save(<-self.createRegistryReportCallbackImpl(), to: self.registryReportCallbackStoragePath)
256+
}
257+
234258
}
235259
}

cadence/contracts/FlowYieldVaultsSchedulerRegistry.cdc

Lines changed: 50 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,11 @@ access(all) contract FlowYieldVaultsSchedulerRegistry {
5858
/// Stored as a dictionary for O(1) add/remove; iteration gives the pending set
5959
access(self) var pendingQueue: {UInt64: Bool}
6060

61+
/// Order for stuck scanning: least recently reported (executed) first.
62+
/// Vaults call reportExecution() on each run (remove id from array, append to end).
63+
/// Supervisor scans only the first MAX_BATCH_SIZE entries for stuck detection.
64+
access(self) var stuckScanOrder: [UInt64]
65+
6166
/* --- ACCOUNT-LEVEL FUNCTIONS --- */
6267

6368
/// Register a YieldVault and store its handler and schedule capabilities (idempotent)
@@ -73,9 +78,27 @@ access(all) contract FlowYieldVaultsSchedulerRegistry {
7378
self.yieldVaultRegistry[yieldVaultID] = true
7479
self.handlerCaps[yieldVaultID] = handlerCap
7580
self.scheduleCaps[yieldVaultID] = scheduleCap
81+
self.stuckScanOrder.append(yieldVaultID)
7682
emit YieldVaultRegistered(yieldVaultID: yieldVaultID)
7783
}
7884

85+
/// Called by the account that holds this contract (e.g. from the wrapper) on every execution. Removes yieldVaultID from stuckScanOrder (if present)
86+
/// and appends it to the end so the Supervisor only scans the first N (least recently executed) for stuck.
87+
access(account) fun reportExecution(yieldVaultID: UInt64) {
88+
if !(self.yieldVaultRegistry[yieldVaultID] ?? false) {
89+
return
90+
}
91+
var i = 0
92+
while i < self.stuckScanOrder.length {
93+
if self.stuckScanOrder[i] == yieldVaultID {
94+
self.stuckScanOrder.remove(at: i)
95+
break
96+
}
97+
i = i + 1
98+
}
99+
self.stuckScanOrder.append(yieldVaultID)
100+
}
101+
79102
/// Adds a yield vault to the pending queue for seeding by the Supervisor
80103
access(account) fun enqueuePending(yieldVaultID: UInt64) {
81104
if self.yieldVaultRegistry[yieldVaultID] == true {
@@ -92,12 +115,20 @@ access(all) contract FlowYieldVaultsSchedulerRegistry {
92115
}
93116
}
94117

95-
/// Unregister a YieldVault (idempotent) - removes from registry, capabilities, and pending queue
118+
/// Unregister a YieldVault (idempotent) - removes from registry, capabilities, pending queue, and stuckScanOrder
96119
access(account) fun unregister(yieldVaultID: UInt64) {
97120
self.yieldVaultRegistry.remove(key: yieldVaultID)
98121
self.handlerCaps.remove(key: yieldVaultID)
99122
self.scheduleCaps.remove(key: yieldVaultID)
100123
let pending = self.pendingQueue.remove(key: yieldVaultID)
124+
var i = 0
125+
while i < self.stuckScanOrder.length {
126+
if self.stuckScanOrder[i] == yieldVaultID {
127+
self.stuckScanOrder.remove(at: i)
128+
break
129+
}
130+
i = i + 1
131+
}
101132
emit YieldVaultUnregistered(yieldVaultID: yieldVaultID, wasInPendingQueue: pending != nil)
102133
}
103134

@@ -156,19 +187,19 @@ access(all) contract FlowYieldVaultsSchedulerRegistry {
156187
/// Get paginated pending yield vault IDs
157188
/// @param page: The page number (0-indexed)
158189
/// @param size: The page size (defaults to MAX_BATCH_SIZE if nil)
159-
access(all) view fun getPendingYieldVaultIDsPaginated(page: Int, size: Int?): [UInt64] {
160-
let pageSize = size ?? self.MAX_BATCH_SIZE
190+
access(all) view fun getPendingYieldVaultIDsPaginated(page: Int, size: UInt?): [UInt64] {
191+
let pageSize = size ?? Int(self.MAX_BATCH_SIZE)
161192
let allPending = self.pendingQueue.keys
162-
let startIndex = page * pageSize
163-
193+
let startIndex = page * Int(pageSize)
194+
164195
if startIndex >= allPending.length {
165196
return []
166197
}
167-
168-
let endIndex = startIndex + pageSize > allPending.length
169-
? allPending.length
170-
: startIndex + pageSize
171-
198+
199+
let endIndex = startIndex + Int(pageSize) > allPending.length
200+
? allPending.length
201+
: startIndex + Int(pageSize)
202+
172203
return allPending.slice(from: startIndex, upTo: endIndex)
173204
}
174205

@@ -177,6 +208,14 @@ access(all) contract FlowYieldVaultsSchedulerRegistry {
177208
return self.pendingQueue.length
178209
}
179210

211+
/// Returns the first n yield vault IDs from the stuck-scan order (least recently executed first).
212+
/// Supervisor should only scan these for stuck detection instead of all registered vaults.
213+
/// @param limit: Maximum number of IDs to return (caller typically passes MAX_BATCH_SIZE)
214+
access(all) view fun getStuckScanCandidates(limit: UInt): [UInt64] {
215+
let end = limit > UInt(self.stuckScanOrder.length) ? self.stuckScanOrder.length : limit
216+
return self.stuckScanOrder.slice(from: 0, upTo: Int(end))
217+
}
218+
180219
/// Get global Supervisor capability, if set
181220
/// NOTE: Access restricted - only used internally by the scheduler
182221
access(account)
@@ -193,6 +232,7 @@ access(all) contract FlowYieldVaultsSchedulerRegistry {
193232
self.handlerCaps = {}
194233
self.scheduleCaps = {}
195234
self.pendingQueue = {}
235+
self.stuckScanOrder = []
196236
}
197237
}
198238

cadence/contracts/FlowYieldVaultsSchedulerV1.cdc

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -186,24 +186,8 @@ access(all) contract FlowYieldVaultsSchedulerV1 {
186186

187187
// STEP 1: State-based detection - scan for stuck yield vaults
188188
if scanForStuck {
189-
// TODO: add pagination - this will inevitably fails and at minimum creates inconsistent execution
190-
// effort between runs
191-
let registeredYieldVaults = FlowYieldVaultsSchedulerRegistry.getRegisteredYieldVaultIDs()
192-
var scanned = 0
193-
for yieldVaultID in registeredYieldVaults {
194-
if scanned >= FlowYieldVaultsSchedulerRegistry.MAX_BATCH_SIZE {
195-
break
196-
}
197-
scanned = scanned + 1
198-
199-
// Skip if already in pending queue
200-
// TODO: This is extremely inefficient - accessing from mapping is preferrable to iterating over
201-
// an array
202-
if FlowYieldVaultsSchedulerRegistry.getPendingYieldVaultIDs().contains(yieldVaultID) {
203-
continue
204-
}
205-
206-
// Check if yield vault is stuck (has recurring config, no active schedule, overdue)
189+
let candidates = FlowYieldVaultsSchedulerRegistry.getStuckScanCandidates(limit: UInt(FlowYieldVaultsSchedulerRegistry.MAX_BATCH_SIZE))
190+
for yieldVaultID in candidates {
207191
if FlowYieldVaultsAutoBalancers.isStuckYieldVault(id: yieldVaultID) {
208192
FlowYieldVaultsSchedulerRegistry.enqueuePending(yieldVaultID: yieldVaultID)
209193
emit StuckYieldVaultDetected(yieldVaultID: yieldVaultID)
@@ -213,7 +197,7 @@ access(all) contract FlowYieldVaultsSchedulerV1 {
213197

214198
// STEP 2: Process pending yield vaults - recover them via Schedule capability
215199
let pendingYieldVaults = FlowYieldVaultsSchedulerRegistry.getPendingYieldVaultIDsPaginated(page: 0, size: nil)
216-
200+
217201
for yieldVaultID in pendingYieldVaults {
218202
// Get Schedule capability for this yield vault
219203
let scheduleCap = FlowYieldVaultsSchedulerRegistry.getScheduleCap(yieldVaultID: yieldVaultID)
@@ -457,7 +441,7 @@ access(all) contract FlowYieldVaultsSchedulerV1 {
457441

458442
// Initialize paths
459443
self.SupervisorStoragePath = /storage/FlowYieldVaultsSupervisor
460-
444+
461445
// Configure Supervisor at deploy time
462446
self.ensureSupervisorConfigured()
463447
}

cadence/tests/scheduled_supervisor_test.cdc

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -913,3 +913,130 @@ fun testInsufficientFundsAndRecovery() {
913913
log("- All ".concat(activeScheduleCount.toString()).concat(" yield vaults have active schedules"))
914914
log("========================================")
915915
}
916+
917+
/// Supervisor batch recovery: 200 stuck vaults, no capacity-probe loop.
918+
///
919+
/// Flow: create 200 yield vaults, run 2 scheduling rounds, drain FLOW so executions fail,
920+
/// wait for vaults to be marked stuck, refund FLOW, schedule the supervisor, then advance
921+
/// time for ceil(200/MAX_BATCH_SIZE)+10 supervisor ticks. Asserts all 200 vaults are
922+
/// recovered (YieldVaultRecovered events), none still stuck, and all have active schedules.
923+
/// The +10 extra ticks are a buffer so every vault is processed despite scheduler timing.
924+
access(all)
925+
fun testSupervisorHandlesManyStuckVaults() {
926+
let n = 200
927+
let maxBatchSize = FlowYieldVaultsSchedulerRegistry.MAX_BATCH_SIZE
928+
929+
if snapshot != getCurrentBlockHeight() {
930+
Test.reset(to: snapshot)
931+
}
932+
933+
// 1. Setup: user, FLOW, and grant
934+
let user = Test.createAccount()
935+
mintFlow(to: user, amount: 100000.0)
936+
grantBeta(flowYieldVaultsAccount, user)
937+
mintFlow(to: flowYieldVaultsAccount, amount: 10000.0)
938+
939+
// 2. Create n yield vaults in batch (Test.executeTransactions)
940+
var i = 0
941+
let tx = Test.Transaction(
942+
code: Test.readFile("../transactions/flow-yield-vaults/create_yield_vault.cdc"),
943+
authorizers: [user.address],
944+
signers: [user],
945+
arguments: [strategyIdentifier, flowTokenIdentifier, 5.0]
946+
)
947+
let txs: [Test.Transaction] = []
948+
while i < n {
949+
txs.append(tx)
950+
i = i + 1
951+
}
952+
let results = Test.executeTransactions(txs)
953+
for result in results {
954+
Test.expect(result, Test.beSucceeded())
955+
}
956+
log("testSupervisorHandlesManyStuckVaults: created \(n.toString()) yield vaults")
957+
958+
let yieldVaultIDs = getYieldVaultIDs(address: user.address)!
959+
Test.assert(yieldVaultIDs.length == n, message: "expected \(n.toString()) vaults, got \(yieldVaultIDs.length.toString())")
960+
961+
// 3. Two scheduling rounds so vaults run once
962+
setMockOraclePrice(signer: flowYieldVaultsAccount, forTokenIdentifier: flowTokenIdentifier, price: 1.5)
963+
setMockOraclePrice(signer: flowYieldVaultsAccount, forTokenIdentifier: yieldTokenIdentifier, price: 1.2)
964+
Test.moveTime(by: 60.0 * 10.0 + 10.0)
965+
Test.commitBlock()
966+
Test.moveTime(by: 60.0 * 10.0 + 10.0)
967+
Test.commitBlock()
968+
969+
// 4. Drain FLOW so subsequent executions fail and vaults become stuck
970+
let balanceBeforeDrain = (executeScript(
971+
"../scripts/flow-yield-vaults/get_flow_balance.cdc",
972+
[flowYieldVaultsAccount.address]
973+
).returnValue! as! UFix64)
974+
if balanceBeforeDrain > 0.01 {
975+
let drainRes = _executeTransaction(
976+
"../transactions/flow-yield-vaults/drain_flow.cdc",
977+
[balanceBeforeDrain - 0.001],
978+
flowYieldVaultsAccount
979+
)
980+
Test.expect(drainRes, Test.beSucceeded())
981+
}
982+
log("testSupervisorHandlesManyStuckVaults: drained FLOW, waiting for vaults to be marked stuck")
983+
984+
// 5. Wait rounds until vaults are marked stuck
985+
var waitRound = 0
986+
while waitRound < 6 {
987+
Test.moveTime(by: 60.0 * 10.0 + 10.0)
988+
Test.commitBlock()
989+
waitRound = waitRound + 1
990+
}
991+
992+
// 6. Refund FLOW and schedule supervisor
993+
mintFlow(to: flowYieldVaultsAccount, amount: 500.0)
994+
Test.commitBlock()
995+
Test.moveTime(by: 1.0)
996+
Test.commitBlock()
997+
998+
let interval = 60.0 * 10.0
999+
let schedSupRes = _executeTransaction(
1000+
"../transactions/flow-yield-vaults/admin/schedule_supervisor.cdc",
1001+
[interval, UInt8(1), UInt64(5000), true],
1002+
flowYieldVaultsAccount
1003+
)
1004+
Test.expect(schedSupRes, Test.beSucceeded())
1005+
1006+
// 7. Advance time for supervisor ticks (ceil(n/MAX_BATCH_SIZE)+10); each tick processes a batch
1007+
let supervisorRunsNeeded = (UInt(n) + UInt(maxBatchSize) - 1) / UInt(maxBatchSize)
1008+
var run = 0 as UInt
1009+
while run < supervisorRunsNeeded + 10 {
1010+
Test.moveTime(by: 60.0 * 10.0 + 10.0)
1011+
Test.commitBlock()
1012+
run = run + 1
1013+
}
1014+
log("testSupervisorHandlesManyStuckVaults: ran \(supervisorRunsNeeded + 10).toString()) supervisor ticks")
1015+
1016+
let recoveredEvents = Test.eventsOfType(Type<FlowYieldVaultsSchedulerV1.YieldVaultRecovered>())
1017+
Test.assert(recoveredEvents.length >= n, message: "expected at least \(n.toString()) recovered, got \(recoveredEvents.length.toString())")
1018+
log("testSupervisorHandlesManyStuckVaults: recovered \(recoveredEvents.length.toString()) vaults")
1019+
1020+
// 8. Health check: none stuck, all have active schedules
1021+
var stillStuck = 0
1022+
var activeCount = 0
1023+
for yieldVaultID in yieldVaultIDs {
1024+
let isStuckRes = executeScript(
1025+
"../scripts/flow-yield-vaults/is_stuck_yield_vault.cdc",
1026+
[yieldVaultID]
1027+
)
1028+
if isStuckRes.returnValue != nil && (isStuckRes.returnValue! as! Bool) {
1029+
stillStuck = stillStuck + 1
1030+
}
1031+
let hasActiveRes = executeScript(
1032+
"../scripts/flow-yield-vaults/has_active_schedule.cdc",
1033+
[yieldVaultID]
1034+
)
1035+
if hasActiveRes.returnValue != nil && (hasActiveRes.returnValue! as! Bool) {
1036+
activeCount = activeCount + 1
1037+
}
1038+
}
1039+
Test.assert(stillStuck == 0, message: "expected 0 stuck, got \(stillStuck.toString())")
1040+
Test.assert(activeCount == n, message: "expected \(n.toString()) active, got \(activeCount.toString())")
1041+
log("testSupervisorHandlesManyStuckVaults: all \(n.toString()) vaults healthy, active schedules: \(activeCount.toString())")
1042+
}

0 commit comments

Comments
 (0)