local jobName = redis.call('SRANDMEMBER', KEYS[1])
if not jobName then return {'', '0', '0'} end
local time = redis.call('TIME')
local now = time[1] * 1000 + math.floor(time[2] / 1000)
local prefix = ARGV[3]
local jobKey = prefix .. 'job:' .. jobName
local waitqKey = prefix .. 'waitq:' .. jobName
local waitingKey = prefix .. 'waiting'
local runningKey = prefix .. 'running'
local jobState = redis.call('HGET', jobKey, 'state')
local deletedRevision = tonumber(redis.call('HGET', jobKey, 'deletedRevision') or '-1')
local batch = tonumber(ARGV[1])
local reaped = 0
local pendingRound = 0
local fanoutRequests = {}
local function usedBudget()
return reaped + math.floor(#fanoutRequests / 3)
end
local function fenced(runKey)
return jobState == 'DELETED'
or tonumber(redis.call('HGET', runKey, 'definitionRevision') or '0') <= deletedRevision
end
local function terminalState(state)
return state == 'SUCCEEDED' or state == 'FAILED' or state == 'DEAD'
or state == 'SKIPPED' or state == 'CANCELLED'
end
local handoff
local function terminate(runId, runKey, expiredLease)
redis.call('HSET', runKey, 'state', 'CANCELLED', 'finishedAt', now, 'resultCode', 'JOB_DELETED')
if expiredLease then redis.call('HSET', runKey, 'errorType', 'LEASE_EXPIRED') end
redis.call('ZREM', KEYS[2], runId)
redis.call('ZREM', KEYS[3], runId)
redis.call('ZREM', waitingKey, runId)
local releasedSlot = false
if redis.call('HGET', runningKey, jobName) == runId then
redis.call('HDEL', runningKey, jobName)
releasedSlot = true
end
redis.call('XADD', KEYS[4], 'MAXLEN', '~', 100000, '*',
'runId', runId, 'state', 'CANCELLED', 'resultCode', 'JOB_DELETED')
redis.call('PEXPIRE', runKey, tonumber(ARGV[2]))
reaped = reaped + 1
if releasedSlot and handoff then handoff() end
end
handoff = function()
local remaining = batch - usedBudget()
if remaining <= 0 then return false end
local limit = math.min(100, remaining)
for index = 1, limit do
local entry = redis.call('ZPOPMIN', waitqKey, 1)
if not entry[1] then return true end
local runId = entry[1]
local runKey = prefix .. 'run:' .. runId
if redis.call('HGET', runKey, 'state') == 'BLOCKED' then
if fenced(runKey) then
terminate(runId, runKey, false)
elseif not redis.call('HGET', runningKey, jobName) then
redis.call('HSET', runKey, 'state', 'QUEUED', 'nextVisibleAt', now)
redis.call('ZADD', KEYS[2], now, runId)
return true
else
redis.call('ZADD', waitqKey, entry[2], runId)
return true
end
end
if reaped >= batch then break end
end
return redis.call('ZCARD', waitqKey) == 0
end
local waitqClean = false
for index = 1, batch do
if usedBudget() >= batch then break end
local entry = redis.call('ZPOPMIN', waitqKey, 1)
if not entry[1] then
waitqClean = true
break
end
local runKey = prefix .. 'run:' .. entry[1]
if not fenced(runKey) then
if redis.call('HGET', runKey, 'state') == 'BLOCKED'
and not redis.call('HGET', runningKey, jobName) then
redis.call('HSET', runKey, 'state', 'QUEUED', 'nextVisibleAt', now)
redis.call('ZADD', KEYS[2], now, entry[1])
else
redis.call('ZADD', waitqKey, entry[2], entry[1])
end
waitqClean = true
break
end
local state = redis.call('HGET', runKey, 'state')
if state == 'BLOCKED' or state == 'QUEUED' or state == 'RETRY_WAIT' then
terminate(entry[1], runKey, false)
end
end
if not waitqClean and redis.call('ZCARD', waitqKey) == 0 then waitqClean = true end
local function scanIndex(zsetKey, cursorField, dirtyField, cleanField, mode)
local cursor = redis.call('HGET', jobKey, cursorField) or '0'
for iteration = 1, 8 do
if usedBudget() >= batch then break end
local scan = redis.call('ZSCAN', zsetKey, cursor, 'COUNT', 500)
cursor = scan[1]
local members = scan[2]
local pageComplete = true
for index = 1, #members, 2 do
if usedBudget() >= batch then
pageComplete = false
break
end
local runId = members[index]
local score = tonumber(members[index + 1])
local runKey = prefix .. 'run:' .. runId
if mode == 'waiting' and redis.call('EXISTS', runKey) == 0 then
redis.call('ZREM', zsetKey, runId)
elseif redis.call('HGET', runKey, 'jobName') == jobName and fenced(runKey) then
local state = redis.call('HGET', runKey, 'state')
if mode == 'leases' then
if score <= now then
terminate(runId, runKey, true)
else
redis.call('HSET', jobKey, dirtyField, '1')
pendingRound = pendingRound + 1
end
elseif state == 'QUEUED' or state == 'RETRY_WAIT' or state == 'BLOCKED' then
terminate(runId, runKey, false)
elseif mode == 'waiting' and (state == 'FANOUT_CREATING' or state == 'WAITING_CHILDREN') then
if score <= now then
local fanoutId = redis.call('HGET', runKey, 'fanoutId') or ''
if fanoutId == '' then
terminate(runId, runKey, false)
else
table.insert(fanoutRequests, runId)
table.insert(fanoutRequests, fanoutId)
table.insert(fanoutRequests, redis.call('HGET', runKey, 'rootAttempt') or '0')
redis.call('HSET', jobKey, dirtyField, '1')
pendingRound = pendingRound + 1
end
else
redis.call('HSET', jobKey, dirtyField, '1')
pendingRound = pendingRound + 1
end
elseif not state or terminalState(state) then
redis.call('ZREM', zsetKey, runId)
else
redis.call('HSET', jobKey, dirtyField, '1')
pendingRound = pendingRound + 1
end
end
end
if not pageComplete then
redis.call('HSET', jobKey, dirtyField, '1', cleanField, '0')
break
elseif cursor == '0' then
redis.call('HSET', jobKey, cleanField,
(redis.call('HGET', jobKey, dirtyField) or '0') == '0' and '1' or '0')
redis.call('HDEL', jobKey, dirtyField)
break
end
end
redis.call('HSET', jobKey, cursorField, cursor)
return cursor
end
scanIndex(KEYS[2], 'reapCursorVisible', 'reapDirtyVisible', 'reapCleanVisible', 'visible')
scanIndex(KEYS[3], 'reapCursorLeases', 'reapDirtyLeases', 'reapCleanLeases', 'leases')
scanIndex(waitingKey, 'reapCursorWaiting', 'reapDirtyWaiting', 'reapCleanWaiting', 'waiting')
local allClean = waitqClean
and (redis.call('HGET', jobKey, 'reapCleanVisible') or '0') == '1'
and (redis.call('HGET', jobKey, 'reapCleanLeases') or '0') == '1'
and (redis.call('HGET', jobKey, 'reapCleanWaiting') or '0') == '1'
if allClean then
redis.call('SREM', KEYS[1], jobName)
redis.call('HDEL', jobKey, 'reapCursorVisible', 'reapCursorLeases', 'reapCursorWaiting',
'reapCleanVisible', 'reapCleanLeases', 'reapCleanWaiting',
'reapDirtyVisible', 'reapDirtyLeases', 'reapDirtyWaiting')
end
local active = (not allClean or reaped > 0 or pendingRound > 0) and '1' or '0'
local response = {jobName, tostring(reaped), active}
for _, value in ipairs(fanoutRequests) do table.insert(response, value) end
return response