Skip to content

Commit 4b53b4c

Browse files
[LIVY-1073] Add SessionManager refresh and /recovery operator endpoint
## What changes were proposed in this pull request? Adds a `SessionManager` refresh capability and a `/recovery` operator endpoint on `LivyServer`, allowing an operator to trigger a manual re-sync of in-memory session state against the recovery/session-state store without restarting the Livy server process. The `/recovery/*` endpoints are implemented as a standalone `RecoveryServlet`, restricted to `livy.superusers`. If `livy.server.recovery.mode` is `off` (no state store configured), the endpoints return `409` rather than a silently-empty result. The three endpoints (`/recovery/sessions/refresh`, `/recovery/batches/refresh`, `/recovery/refresh`) are documented in `docs/rest-api.md`. ## How was this patch tested? Unit tests added in `SessionManagerSpec` covering the refresh path, and `RecoveryServletSpec` covering the servlet's authorization (403 vs 200) and recovery-mode-off (409) behavior. ## Was this patch authored or co-authored using generative AI tooling? Yes. Generated-by: Claude Code (Sonnet 5, Anthropic), used to scan/plan the extraction of local patches for upstreaming and to draft/iterate on the code and tests in this PR under human review. Please refer to the [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html) for details. Co-authored-by: Venugopal Allenki <vallenki@salesforce.com>
1 parent 7205db3 commit 4b53b4c

6 files changed

Lines changed: 406 additions & 2 deletions

File tree

‎docs/rest-api.md‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,6 +696,49 @@ instance.
696696
</tr>
697697
</table>
698698

699+
### POST /recovery/sessions/refresh
700+
701+
Operator endpoint. Re-scans the recovery state store and imports any interactive
702+
sessions written by another Livy server that this server doesn't yet have in memory.
703+
Add-only: existing in-memory sessions are never removed by this call.
704+
705+
**Authorization:** restricted to users listed in `livy.superusers`; all other callers
706+
get a `403`. Also requires `livy.server.recovery.mode` to be set to something other
707+
than `off` (i.e. a real state store must be configured); otherwise this returns `409`,
708+
since there would be nothing to refresh from.
709+
710+
#### Response Body
711+
712+
<table class="table">
713+
<tr><th>Name</th><th>Description</th><th>Type</th></tr>
714+
<tr>
715+
<td>added</td>
716+
<td>Number of sessions imported from the state store that weren't already in memory</td>
717+
<td>int</td>
718+
</tr>
719+
<tr>
720+
<td>total</td>
721+
<td>Total number of interactive sessions in memory after the refresh</td>
722+
<td>int</td>
723+
</tr>
724+
<tr>
725+
<td>failed</td>
726+
<td>Number of state store entries that failed to deserialize</td>
727+
<td>int</td>
728+
</tr>
729+
</table>
730+
731+
### POST /recovery/batches/refresh
732+
733+
Same as `POST /recovery/sessions/refresh`, but for batch sessions. Same authorization
734+
and recovery-mode requirements, and the same response body shape.
735+
736+
### POST /recovery/refresh
737+
738+
Runs both of the above in a single call. Returns a JSON object with `sessions` and
739+
`batches` keys, each holding a response body of the same shape as the individual
740+
endpoints above.
741+
699742
## REST Objects
700743

701744
### Session

‎server/src/main/scala/org/apache/livy/server/LivyServer.scala‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ import org.apache.livy._
3939
import org.apache.livy.server.auth.LdapAuthenticationHandlerImpl
4040
import org.apache.livy.server.batch.BatchSessionServlet
4141
import org.apache.livy.server.interactive.InteractiveSessionServlet
42-
import org.apache.livy.server.recovery.{SessionStore, StateStore, ZooKeeperManager}
42+
import org.apache.livy.server.recovery.{RecoveryServlet, SessionStore, StateStore, ZooKeeperManager}
4343
import org.apache.livy.server.ui.UIServlet
4444
import org.apache.livy.sessions.{BatchSessionManager, InteractiveSessionManager}
4545
import org.apache.livy.sessions.SessionManager.SESSION_RECOVERY_MODE_OFF
@@ -176,6 +176,11 @@ class LivyServer extends Logging {
176176
}
177177
}
178178

179+
// Operator endpoint: re-scan the recovery state store and import any sessions
180+
// written by another Livy server. Guarded by livy.superusers.
181+
val recoveryServlet = new RecoveryServlet(
182+
livyConf, accessManager, interactiveSessionManager, batchSessionManager)
183+
179184
// Servlet for hosting static files such as html, css, and js
180185
// Necessary since Jetty cannot set it's resource base inside a jar
181186
// Returns 404 if the file does not exist
@@ -258,6 +263,8 @@ class LivyServer extends Logging {
258263
}
259264

260265
mount(context, livyVersionServlet, "/version/*")
266+
267+
mount(context, recoveryServlet, "/recovery/*")
261268
} catch {
262269
case e: Throwable =>
263270
error("Exception thrown when initializing server", e)
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.livy.server.recovery
19+
20+
import javax.servlet.http.HttpServletRequest
21+
22+
import org.apache.livy.{LivyConf, Logging}
23+
import org.apache.livy.server.{AccessManager, JsonServlet}
24+
import org.apache.livy.sessions.{BatchSessionManager, InteractiveSessionManager}
25+
import org.apache.livy.sessions.SessionManager.{RefreshResult, SESSION_RECOVERY_MODE_OFF}
26+
27+
/**
28+
* Operator endpoint that re-scans the recovery state store and imports any sessions
29+
* written by another Livy server. Guarded by `livy.superusers`.
30+
*/
31+
class RecoveryServlet(
32+
livyConf: LivyConf,
33+
accessManager: AccessManager,
34+
interactiveSessionManager: InteractiveSessionManager,
35+
batchSessionManager: BatchSessionManager)
36+
extends JsonServlet
37+
with Logging {
38+
39+
protected def remoteUser(req: HttpServletRequest): String = req.getRemoteUser()
40+
41+
before() {
42+
contentType = "application/json"
43+
val user = remoteUser(request)
44+
if (!accessManager.checkSuperUser(user)) {
45+
halt(403, Map("msg" -> s"User '$user' not authorized for recovery endpoints."))
46+
}
47+
if (livyConf.get(LivyConf.RECOVERY_MODE) == SESSION_RECOVERY_MODE_OFF) {
48+
halt(409, Map("msg" ->
49+
"Recovery is disabled (livy.server.recovery.mode=off); there is no state to refresh."))
50+
}
51+
}
52+
53+
private def resultMap(r: RefreshResult): Map[String, Int] =
54+
Map("added" -> r.added, "total" -> r.total, "failed" -> r.failed)
55+
56+
post("/sessions/refresh") {
57+
info(s"Interactive session refresh triggered by user='${remoteUser(request)}'")
58+
resultMap(interactiveSessionManager.refresh())
59+
}
60+
61+
post("/batches/refresh") {
62+
info(s"Batch session refresh triggered by user='${remoteUser(request)}'")
63+
resultMap(batchSessionManager.refresh())
64+
}
65+
66+
post("/refresh") {
67+
info(s"Full session refresh triggered by user='${remoteUser(request)}'")
68+
Map(
69+
"batches" -> resultMap(batchSessionManager.refresh()),
70+
"sessions" -> resultMap(interactiveSessionManager.refresh())
71+
)
72+
}
73+
74+
}

‎server/src/main/scala/org/apache/livy/sessions/SessionManager.scala‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ import org.apache.livy.sessions.Session.RecoveryMetadata
3636
object SessionManager {
3737
val SESSION_RECOVERY_MODE_OFF = "off"
3838
val SESSION_RECOVERY_MODE_RECOVERY = "recovery"
39+
40+
/** Outcome of a [[SessionManager.refresh]] call. */
41+
case class RefreshResult(added: Int, total: Int, failed: Int)
3942
}
4043

4144
class BatchSessionManager(
@@ -100,8 +103,14 @@ class SessionManager[S <: Session, R <: RecoveryMetadata : ClassTag](
100103
}
101104

102105
def register(session: S): S = {
103-
info(s"Registering new session ${session.id}")
104106
synchronized {
107+
sessions.get(session.id) match {
108+
case Some(existing) =>
109+
debug(s"Session ${session.id} already registered; skipping duplicate registration.")
110+
return existing
111+
case None =>
112+
}
113+
info(s"Registering new session ${session.id}")
105114
session.name.foreach { sessionName =>
106115
if (sessionsByName.contains(sessionName)) {
107116
val errMsg = s"Duplicate session name: ${session.name} for session ${session.id}"
@@ -229,6 +238,38 @@ class SessionManager[S <: Session, R <: RecoveryMetadata : ClassTag](
229238
recoveredSessions
230239
}
231240

241+
/**
242+
* Re-scan the state store and import sessions written by another Livy server.
243+
* Add-only: in-memory sessions are not removed even if their state-store entry is
244+
* gone. The id counter is advanced forward only.
245+
*/
246+
def refresh(): RefreshResult = {
247+
// Read the state store outside the SessionManager monitor so we don't block the
248+
// garbage collector and heartbeat watchdog while doing N small EFS / ZK reads.
249+
val storeNextId = sessionStore.getNextSessionId(sessionType)
250+
val sessionMetadata = sessionStore.getAllSessions[R](sessionType)
251+
252+
val recoveryFailure = sessionMetadata.filter(_.isFailure).map(_.failed.get)
253+
recoveryFailure.foreach(ex => warn(s"Refresh failure for $sessionType: ${ex.getMessage}", ex))
254+
255+
synchronized {
256+
if (storeNextId > idCounter.get) {
257+
idCounter.set(storeNextId)
258+
}
259+
260+
val before = sessions.size
261+
sessionMetadata.flatMap(_.toOption)
262+
.filterNot(m => sessions.contains(m.id))
263+
.map(sessionRecovery)
264+
.foreach(register)
265+
266+
val added = sessions.size - before
267+
info(s"Refreshed $sessionType sessions: added=$added, total=${sessions.size}," +
268+
s" failed=${recoveryFailure.size}, next session id=$idCounter")
269+
RefreshResult(added, sessions.size, recoveryFailure.size)
270+
}
271+
}
272+
232273
private class GarbageCollector extends Thread("session gc thread") {
233274

234275
setDaemon(true)
Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.livy.server.recovery
19+
20+
import javax.servlet.http.HttpServletRequest
21+
22+
import org.mockito.Mockito.when
23+
import org.scalatestplus.mockito.MockitoSugar.mock
24+
25+
import org.apache.livy.LivyConf
26+
import org.apache.livy.server.{AccessManager, BaseJsonServletSpec}
27+
import org.apache.livy.server.batch.BatchRecoveryMetadata
28+
import org.apache.livy.server.interactive.InteractiveRecoveryMetadata
29+
import org.apache.livy.sessions.{BatchSessionManager, InteractiveSessionManager}
30+
import org.apache.livy.sessions.SessionManager.{SESSION_RECOVERY_MODE_OFF,
31+
SESSION_RECOVERY_MODE_RECOVERY}
32+
33+
object RecoveryServletSpec {
34+
val REMOTE_USER_HEADER = "X-Livy-RecoveryServlet-User"
35+
val ADMIN = "__admin__"
36+
val REGULAR_USER = "__user__"
37+
}
38+
39+
/** Reads the test-only remote-user header instead of the (unavailable in tests) container user. */
40+
class TestRecoveryServlet(
41+
livyConf: LivyConf,
42+
accessManager: AccessManager,
43+
interactiveSessionManager: InteractiveSessionManager,
44+
batchSessionManager: BatchSessionManager)
45+
extends RecoveryServlet(livyConf, accessManager, interactiveSessionManager, batchSessionManager) {
46+
47+
override protected def remoteUser(req: HttpServletRequest): String = {
48+
req.getHeader(RecoveryServletSpec.REMOTE_USER_HEADER)
49+
}
50+
}
51+
52+
trait RecoveryServletSpecBase extends BaseJsonServletSpec {
53+
54+
import RecoveryServletSpec._
55+
56+
protected def recoveryMode: String
57+
58+
protected def headersFor(user: String): Map[String, String] =
59+
defaultHeaders ++ Map(REMOTE_USER_HEADER -> user)
60+
61+
private def mockSessionStore(): SessionStore = {
62+
val sessionStore = mock[SessionStore]
63+
when(sessionStore.getAllSessions[BatchRecoveryMetadata]("batch")).thenReturn(Seq.empty)
64+
when(sessionStore.getAllSessions[InteractiveRecoveryMetadata]("interactive"))
65+
.thenReturn(Seq.empty)
66+
when(sessionStore.getNextSessionId("batch")).thenReturn(0)
67+
when(sessionStore.getNextSessionId("interactive")).thenReturn(0)
68+
sessionStore
69+
}
70+
71+
private val livyConf = new LivyConf()
72+
.set(LivyConf.SUPERUSERS, ADMIN)
73+
.set(LivyConf.RECOVERY_MODE, recoveryMode)
74+
private val accessManager = new AccessManager(livyConf)
75+
private val sessionStore = mockSessionStore()
76+
private val batchSessionManager = new BatchSessionManager(livyConf, sessionStore)
77+
private val interactiveSessionManager = new InteractiveSessionManager(livyConf, sessionStore)
78+
79+
addServlet(
80+
new TestRecoveryServlet(livyConf, accessManager, interactiveSessionManager,
81+
batchSessionManager),
82+
"/*")
83+
}
84+
85+
class RecoveryServletEnabledSpec extends RecoveryServletSpecBase {
86+
87+
import RecoveryServletSpec._
88+
89+
override protected def recoveryMode: String = SESSION_RECOVERY_MODE_RECOVERY
90+
91+
describe("RecoveryServlet with recovery enabled") {
92+
93+
it("rejects non-superusers with 403") {
94+
post("/refresh", headers = headersFor(REGULAR_USER)) {
95+
status should be (403)
96+
}
97+
}
98+
99+
it("allows superusers and returns refresh counts with 200") {
100+
post("/refresh", headers = headersFor(ADMIN)) {
101+
status should be (200)
102+
body should include ("\"sessions\"")
103+
body should include ("\"batches\"")
104+
}
105+
}
106+
107+
it("allows superusers on the single-manager endpoints with 200") {
108+
post("/sessions/refresh", headers = headersFor(ADMIN)) {
109+
status should be (200)
110+
body should include ("\"added\"")
111+
}
112+
post("/batches/refresh", headers = headersFor(ADMIN)) {
113+
status should be (200)
114+
body should include ("\"added\"")
115+
}
116+
}
117+
}
118+
}
119+
120+
class RecoveryServletDisabledSpec extends RecoveryServletSpecBase {
121+
122+
import RecoveryServletSpec._
123+
124+
override protected def recoveryMode: String = SESSION_RECOVERY_MODE_OFF
125+
126+
describe("RecoveryServlet with recovery disabled") {
127+
128+
it("returns 409 when recovery is disabled, even for superusers") {
129+
post("/refresh", headers = headersFor(ADMIN)) {
130+
status should be (409)
131+
}
132+
}
133+
134+
it("checks authorization before the recovery-mode check") {
135+
post("/refresh", headers = headersFor(REGULAR_USER)) {
136+
status should be (403)
137+
}
138+
}
139+
}
140+
}

0 commit comments

Comments
 (0)