diff --git a/MODULE.bazel b/MODULE.bazel index 4db5cf9b..3a6a46c2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,6 +1,6 @@ module( name = "android_env", - version = "1.3.0", + version = "2.0.0", ) # Core rules diff --git a/android_env/BUILD.bazel b/android_env/BUILD.bazel index 6a55a385..d76ed71d 100644 --- a/android_env/BUILD.bazel +++ b/android_env/BUILD.bazel @@ -8,7 +8,6 @@ py_library( srcs = ["env_interface.py"], deps = [ "//android_env/proto:adb_py_pb2", - "//android_env/proto:state_py_pb2", "@pip//dm_env", "@pip//numpy", ], @@ -55,7 +54,6 @@ py_package( "//android_env/mcp:android_mcp_tools", "//android_env/playground:playground_lib", "//android_env/rl:environment", - "//android_env/wrappers:a11y_grpc_wrapper", "//android_env/wrappers:discrete_action_wrapper", "//android_env/wrappers:flat_interface_wrapper", "//android_env/wrappers:float_pixels_wrapper", @@ -74,23 +72,22 @@ py_wheel( author = "DeepMind", distribution = "android_env", extra_requires = { + "gym": ["gymnasium>=0.29.1"], "mcp": ["mcp>=1.26,<2"], "playground": ["ml-collections"], }, license = "Apache License, Version 2.0", requires = [ - "absl-py>=0.1.0", + "absl-py>=2.1.0", "dm-env>=1.6", - "numpy>=1.26.0", + "numpy>=2.1.0", "portpicker>=1.6.0", "pillow>=10.4.0", - "gymnasium>=0.29.1", - "protobuf>=5.27.2", - "grpcio>=1.64.2", + "protobuf>=6,<8", "opencv-python-headless>=4.10.0", "websocket-client>=1.8.0", ], summary = "AndroidEnv: A Gym/Gymnasium Environment for Android", - version = "1.3.0", + version = "2.0.0", deps = [":android_env_pkg"], ) diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarder.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarder.kt deleted file mode 100644 index a2aba7e6..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarder.kt +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright 2026 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.androidenv.accessibilityforwarder - -import android.accessibilityservice.AccessibilityService -import android.util.Log -import android.view.accessibility.AccessibilityEvent -import android.view.accessibility.AccessibilityNodeInfo -import android.view.accessibility.AccessibilityWindowInfo -import com.google.androidenv.accessibilityforwarder.A11yServiceGrpcKt.A11yServiceCoroutineStub -import io.grpc.ManagedChannel -import io.grpc.ManagedChannelBuilder -import io.grpc.ProxyDetector -import io.grpc.StatusException -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.withTimeout - -/** - * An Android service that listens to accessibility events and forwards them via gRPC. - * - * This service also logs the accessibility tree if [LogFlags.logAccessibilityTree] is set and if - * [LogFlags.grpcPort] is positive. - * - * Please see - * https://developer.android.com/reference/android/view/accessibility/AccessibilityEvent#getEventType() - * for a comprehensive list of events emitted by Android. - */ -class AccessibilityForwarder( - private val channelFactory: (host: String, port: Int) -> ManagedChannel = { host, port -> - ManagedChannelBuilder.forAddress(host, port) - .proxyDetector(ProxyDetector { _ -> null }) - .usePlaintext() - .build() - } -) : AccessibilityService() { - - init { - // Spawn long-running thread for periodically logging the tree. - Thread( - Runnable { - while (LogFlags.a11yTreePeriodMs > 0) { - try { - logAccessibilityTree() - } catch (e: ConcurrentModificationException) { - continue - } - - Thread.sleep(/* millis= */ LogFlags.a11yTreePeriodMs) - } - } - ) - .start() - } - - // grpcStub has a backing property that can be reset to null. - private var _grpcStub: A11yServiceCoroutineStub? = null - val grpcStub: A11yServiceCoroutineStub - get() { - if (_grpcStub == null) { - Log.i(TAG, "Building channel on ${LogFlags.grpcHost}:${LogFlags.grpcPort}.") - _grpcStub = A11yServiceCoroutineStub(channelFactory(LogFlags.grpcHost, LogFlags.grpcPort)) - } - return _grpcStub!! - } - - private fun resetGrpcStub() { - _grpcStub = null - } - - override fun onInterrupt() { - LogFlags.a11yTreePeriodMs = 0 // Turn off periodic tree forwarding. - } - - override fun onAccessibilityEvent(event: AccessibilityEvent?) { - if (event == null) { - Log.i(TAG, "`event` is null.") - return - } - - logExtrasForEvent(event) - val eventType = event.eventType - val eventTypeStr: String = AccessibilityEvent.eventTypeToString(eventType) - if (eventTypeStr.isNotEmpty()) { - Log.i(TAG, eventTypeStr) - } - } - - private fun logAccessibilityTree() { - if (!LogFlags.logAccessibilityTree) { - Log.i(TAG, "Not logging accessibility tree") - return - } - - val windows = getWindowsOrNull() - - if (windows == null) { - Log.i(TAG, "windows is null.") - return - } - - // Check gRPC port before actually building the forest. - if (LogFlags.grpcPort <= 0) { - Log.w(TAG, "Can't log accessibility tree because gRPC port has not been set.") - return - } - - val forest = creator.buildForest(windows) - try { - val grpcTimeoutMillis = 1000L - val response: ForestResponse = - with(grpcStub) { - Log.i(TAG, "sending (blocking) gRPC request for tree.") - runBlocking { withTimeout(grpcTimeoutMillis) { sendForest(forest) } } - } - if (response.error.isNotEmpty()) { - Log.w(TAG, "gRPC response.error: ${response.error}") - } else { - Log.i(TAG, "gRPC request for tree succeeded.") - } - } catch (e: StatusException) { - Log.w(TAG, "gRPC StatusException; are you sure networking is turned on?") - Log.i(TAG, "extra: exception ['$e']") - resetGrpcStub() - } catch (e: TimeoutCancellationException) { - Log.w(TAG, "gRPC TimeoutCancellationException; are you sure networking is turned on?") - Log.i(TAG, "extra: exception ['$e']") - resetGrpcStub() - } - } - - private fun getWindowsOrNull(): List? = - try { - windows - } catch (e: NullPointerException) { - null - } - - /** Logs extras for all event types. */ - private fun logExtrasForEvent(event: AccessibilityEvent) { - - val events: MutableMap = mutableMapOf() - - val sourceDescription = event.source?.contentDescription() - if (!sourceDescription.isNullOrEmpty()) { - events.put("source_content_description", sourceDescription) - } - - // Output the event text. - val eventText = event.text.joinToString(", ") - if (eventText.isNotEmpty()) { - events.put("event_text", eventText) - } - - // Output the source text. - val sourceText = event.source?.text?.toString() - if (!sourceText.isNullOrEmpty()) { - events.put("source_text", sourceText) - } - - val eventTypeStr: String = AccessibilityEvent.eventTypeToString(event.eventType) - if (eventTypeStr.isNotEmpty()) { - events.put("event_type", eventTypeStr) - } - - val className = event.source?.className?.toString() - if (!className.isNullOrEmpty()) { - events.put("source_class_name", className) - } - - val packageName = event.packageName?.toString() - if (!packageName.isNullOrEmpty()) { - events.put("event_package_name", packageName) - } - - // Text editing properties. - val beforeText = event.beforeText?.toString() - if (!beforeText.isNullOrEmpty()) { - events.put("before_text", beforeText) - } - - val fromIndex = event.fromIndex - if (fromIndex != -1) { - events.put("from_index", fromIndex.toString()) - } - - val toIndex = event.toIndex - if (toIndex != -1) { - events.put("to_index", toIndex.toString()) - } - - val addedCount = event.addedCount - if (addedCount != -1) { - events.put("added_count", addedCount.toString()) - } - - val removedCount = event.removedCount - if (removedCount != -1) { - events.put("removed_count", removedCount.toString()) - } - - // Text traversal properties - val movementGranularity = event.movementGranularity - if (movementGranularity != 0) { - events.put("movement_granularity", movementGranularity.toString()) - } - - val action = event.action - if (action != 0) { - events.put("action", action.toString()) - } - - // Scrolling properties. - if (eventTypeStr == "TYPE_VIEW_SCROLLED") { - events.put("scroll_delta_x", event.scrollDeltaX.toString()) - events.put("scroll_delta_y", event.scrollDeltaY.toString()) - } - - // Report viewID so we know exactly where the event came from. - val viewId = event.source?.viewIdResourceName?.toString() - if (!viewId.isNullOrEmpty()) { - events.put("view_id", viewId) - } - - // Format [events] as a Python dict. - if (events.isNotEmpty()) { - events.put("event_timestamp_ms", event.eventTime.toString(10)) - // Check if we want to use gRPC. - if (LogFlags.grpcPort > 0) { - try { - val grpcTimeoutMillis = 1000L - val request = eventRequest { this.event.putAll(events) } - val response: EventResponse = - with(grpcStub) { - Log.i(TAG, "sending (blocking) gRPC request for event.") - runBlocking { withTimeout(grpcTimeoutMillis) { sendEvent(request) } } - } - if (response.error.isNotEmpty()) { - Log.w(TAG, "gRPC response.error: ${response.error}") - } else { - Log.i(TAG, "gRPC request for event succeeded.") - } - } catch (e: StatusException) { - Log.w(TAG, "gRPC StatusException; are you sure networking is turned on?") - Log.i(TAG, "extra: exception ['$e']") - resetGrpcStub() - } catch (e: TimeoutCancellationException) { - Log.w(TAG, "gRPC TimeoutCancellationException; are you sure networking is turned on?") - Log.i(TAG, "extra: exception ['$e']") - resetGrpcStub() - } - } else { - Log.w(TAG, "Can't log accessibility event because gRPC port has not been set.") - } - } - } - - /** Recursively climbs the accessibility tree until the root, collecting descriptions. */ - private fun AccessibilityNodeInfo?.contentDescription(): String { - if (this == null) { - return "" - } - - val descriptions = mutableListOf() - var current: AccessibilityNodeInfo? = this - while (current != null) { - val description = current.contentDescription - if (description != null) { - descriptions.add(description.toString()) - } - - current = current.parent - } - return descriptions.joinToString(", ") - } - - companion object { - private const val TAG = "AndroidRLTask" - private val creator = AccessibilityTreeCreator() - } -} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarderTest.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarderTest.kt deleted file mode 100644 index 4ad29367..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityForwarderTest.kt +++ /dev/null @@ -1,516 +0,0 @@ -// Copyright 2026 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.androidenv.accessibilityforwarder - -import android.view.accessibility.AccessibilityEvent -import android.view.accessibility.AccessibilityNodeInfo -import android.view.accessibility.AccessibilityWindowInfo -import com.google.common.truth.Truth.assertThat -import io.grpc.Status -import io.grpc.StatusException -import io.grpc.inprocess.InProcessChannelBuilder -import io.grpc.inprocess.InProcessServerBuilder -import io.grpc.testing.GrpcCleanupRule -import org.junit.Assert.assertFalse -import org.junit.Rule -import org.junit.Test -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestParameterInjector -import org.robolectric.Shadows.shadowOf - -@RunWith(RobolectricTestParameterInjector::class) -class AccessibilityForwarderTest { - - @get:Rule(order = 1) val cleanupRule = GrpcCleanupRule() - - class FakeAccessibilityService : A11yServiceGrpcKt.A11yServiceCoroutineImplBase() { - var sendForestChecker: (AndroidAccessibilityForest) -> String = { _ -> "" } - var sendEventChecker: (EventRequest) -> String = { _ -> "" } - - override suspend fun sendForest(request: AndroidAccessibilityForest) = forestResponse { - error = sendForestChecker(request) - } - - override suspend fun sendEvent(request: EventRequest) = eventResponse { - error = sendEventChecker(request) - } - } - - protected lateinit var forwarder: AccessibilityForwarder - protected val fakeA11yService = FakeAccessibilityService() - protected val channel by lazy { - val serverName: String = InProcessServerBuilder.generateName() - cleanupRule.register( - InProcessServerBuilder.forName(serverName) - .directExecutor() - .addService(fakeA11yService) - .build() - .start() - ) - cleanupRule.register(InProcessChannelBuilder.forName(serverName).directExecutor().build()) - } - - /** Initializes [forwarder] and [LogFlags] from the given args. */ - fun createForwarder( - logAccessibilityTree: Boolean = false, - a11yTreePeriodMs: Long = 0, - grpcHost: String = "10.0.2.2", - grpcPort: Int = 0, - a11yWindows: MutableList? = null, - ) { - LogFlags.logAccessibilityTree = logAccessibilityTree - LogFlags.a11yTreePeriodMs = a11yTreePeriodMs - LogFlags.grpcHost = grpcHost - LogFlags.grpcPort = grpcPort - forwarder = AccessibilityForwarder({ _, _ -> channel }) - if (a11yWindows == null) { - shadowOf(forwarder).setWindows(mutableListOf(AccessibilityWindowInfo.obtain())) - } else { - shadowOf(forwarder).setWindows(a11yWindows) - } - } - - @Test - fun onInterrupt_doesNotCrash() { - // Arrange. - createForwarder(logAccessibilityTree = false) - fakeA11yService.sendEventChecker = { _: EventRequest -> - assertFalse(true) // This should not be called. - "" // This should be unreachable - } - - // Act. - forwarder.onInterrupt() - - // Assert. - // See `sendEventChecker` above. - } - - @Test - fun onAccessibilityEvent_nullEventShouldBeIgnored() { - // Arrange. - createForwarder(logAccessibilityTree = false) - fakeA11yService.sendEventChecker = { _: EventRequest -> - assertFalse(true) // This should not be called. - "" // This should be unreachable - } - - // Act. - forwarder.onAccessibilityEvent(null) - - // Assert. - // See `sendEventChecker` above. - } - - @Test - fun onAccessibilityEvent_knownEventWithNoInformationShouldNotBeEmitted() { - // Arrange. - createForwarder(logAccessibilityTree = false) - var nodeInfo = AccessibilityNodeInfo() - nodeInfo.setContentDescription("") - var event = AccessibilityEvent() - shadowOf(event).setSourceNode(nodeInfo) - fakeA11yService.sendEventChecker = { _: EventRequest -> - assertFalse(true) // This should not be called. - "" // This should be unreachable - } - - // Act. - forwarder.onAccessibilityEvent(event) - - // Assert. - // See `sendEventChecker` above. - } - - @Test - fun onAccessibilityEvent_typeViewClicked_sendEventViaGrpc() { - // Arrange. - createForwarder(logAccessibilityTree = false, grpcPort = 1234) - forwarder = AccessibilityForwarder({ _, _ -> channel }) - var nodeInfo = AccessibilityNodeInfo() - nodeInfo.setContentDescription("My Content Description") - nodeInfo.setText("My Source Text") - nodeInfo.setClassName("AwesomeClass") - var event = AccessibilityEvent() - event.setEventTime(1357924680) - event.setEventType(AccessibilityEvent.TYPE_VIEW_CLICKED) - event.getText().add("Some text!") - event.setPackageName("some.loooong.package.name") - shadowOf(event).setSourceNode(nodeInfo) - fakeA11yService.sendEventChecker = { request: EventRequest -> - // Check that all fields are consistent with how they were set above. - assertThat(request.eventMap.get("event_type")).isEqualTo("TYPE_VIEW_CLICKED") - assertThat(request.eventMap.get("event_package_name")).isEqualTo("some.loooong.package.name") - assertThat(request.eventMap.get("source_content_description")) - .isEqualTo("My Content Description") - assertThat(request.eventMap.get("source_text")).isEqualTo("My Source Text") - assertThat(request.eventMap.get("source_class_name")).isEqualTo("AwesomeClass") - assertThat(request.eventMap.get("event_text")).isEqualTo("Some text!") - assertThat(request.eventMap.get("event_timestamp_ms")).isEqualTo("1357924680") - // No error message - "" - } - - // Act. - forwarder.onAccessibilityEvent(event) - - // Assert. - // See `sendEventChecker` above. - } - - @Test - fun onAccessibilityEvent_typeViewTextChanged_ensureAllFieldsForwarded() { - // Arrange. - createForwarder(logAccessibilityTree = false, grpcPort = 1234) - var nodeInfo = AccessibilityNodeInfo() - nodeInfo.setContentDescription("My Content Description") - nodeInfo.setText("My Source Text") - nodeInfo.setClassName("AwesomeClass") - var event = AccessibilityEvent() - event.setEventTime(1357924680) - event.setEventType(AccessibilityEvent.TYPE_VIEW_TEXT_CHANGED) - event.getText().add("Some text!") - event.fromIndex = 7 - event.beforeText = "Old words" - event.addedCount = 12 - event.removedCount = 9 - event.setPackageName("some.loooong.package.name") - shadowOf(event).setSourceNode(nodeInfo) - fakeA11yService.sendEventChecker = { request: EventRequest -> - // Check that all fields are consistent with how they were set above. - assertThat(request.eventMap.get("event_type")).isEqualTo("TYPE_VIEW_TEXT_CHANGED") - assertThat(request.eventMap.get("event_package_name")).isEqualTo("some.loooong.package.name") - assertThat(request.eventMap.get("source_content_description")) - .isEqualTo("My Content Description") - assertThat(request.eventMap.get("source_text")).isEqualTo("My Source Text") - assertThat(request.eventMap.get("source_class_name")).isEqualTo("AwesomeClass") - assertThat(request.eventMap.get("event_text")).isEqualTo("Some text!") - assertThat(request.eventMap.get("event_timestamp_ms")).isEqualTo("1357924680") - assertThat(request.eventMap.get("from_index")).isEqualTo("7") - assertThat(request.eventMap.get("before_text")).isEqualTo("Old words") - assertThat(request.eventMap.get("added_count")).isEqualTo("12") - assertThat(request.eventMap.get("removed_count")).isEqualTo("9") - assertFalse(request.eventMap.containsKey("to_index")) - assertFalse(request.eventMap.containsKey("view_id")) - assertFalse(request.eventMap.containsKey("action")) - assertFalse(request.eventMap.containsKey("movement_granularity")) - assertFalse(request.eventMap.containsKey("scroll_delta_x")) - assertFalse(request.eventMap.containsKey("scroll_delta_y")) - // No error message - "" - } - - // Act. - forwarder.onAccessibilityEvent(event) - - // Assert. - // See `sendEventChecker` above. - } - - @Test - fun onAccessibilityEvent_typeViewScrolled_ensureAllFieldsForwarded() { - // Arrange. - createForwarder(logAccessibilityTree = false, grpcPort = 1234) - var nodeInfo = AccessibilityNodeInfo() - nodeInfo.setContentDescription("My Content Description") - nodeInfo.setText("My Source Text") - nodeInfo.setClassName("AwesomeClass") - var event = AccessibilityEvent() - event.setEventTime(1357924680) - event.setEventType(AccessibilityEvent.TYPE_VIEW_SCROLLED) - event.getText().add("Some text!") - event.scrollDeltaX = 13 - event.scrollDeltaY = 27 - event.setPackageName("some.loooong.package.name") - shadowOf(event).setSourceNode(nodeInfo) - fakeA11yService.sendEventChecker = { request: EventRequest -> - // Check that all fields are consistent with how they were set above. - assertThat(request.eventMap.get("event_type")).isEqualTo("TYPE_VIEW_SCROLLED") - assertThat(request.eventMap.get("event_package_name")).isEqualTo("some.loooong.package.name") - assertThat(request.eventMap.get("source_content_description")) - .isEqualTo("My Content Description") - assertThat(request.eventMap.get("source_text")).isEqualTo("My Source Text") - assertThat(request.eventMap.get("source_class_name")).isEqualTo("AwesomeClass") - assertThat(request.eventMap.get("event_text")).isEqualTo("Some text!") - assertThat(request.eventMap.get("event_timestamp_ms")).isEqualTo("1357924680") - assertThat(request.eventMap.get("scroll_delta_x")).isEqualTo("13") - assertThat(request.eventMap.get("scroll_delta_y")).isEqualTo("27") - assertFalse(request.eventMap.containsKey("from_index")) - assertFalse(request.eventMap.containsKey("to_index")) - assertFalse(request.eventMap.containsKey("before_text")) - assertFalse(request.eventMap.containsKey("added_count")) - assertFalse(request.eventMap.containsKey("removed_count")) - // No error message - "" - } - - // Act. - forwarder.onAccessibilityEvent(event) - - // Assert. - // See `sendEventChecker` above. - } - - @Test - fun onAccessibilityEvent_typeViewTextTraversedAtMovementGranularity_ensureAllFieldsForwarded() { - // Arrange. - createForwarder(logAccessibilityTree = false, grpcPort = 1234) - var nodeInfo = AccessibilityNodeInfo() - nodeInfo.setContentDescription("My Content Description") - nodeInfo.setText("My Source Text") - nodeInfo.setClassName("AwesomeClass") - nodeInfo.viewIdResourceName = "this.big.old.view.id" - var event = AccessibilityEvent() - event.setEventTime(1357924680) - event.setEventType(AccessibilityEvent.TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY) - event.getText().add("Some text!") - event.setPackageName("some.loooong.package.name") - event.movementGranularity = 5 - event.fromIndex = 6 - event.toIndex = 8 - event.action = 23 - shadowOf(event).setSourceNode(nodeInfo) - fakeA11yService.sendEventChecker = { request: EventRequest -> - // Check that all fields are consistent with how they were set above. - assertThat(request.eventMap.get("event_type")) - .isEqualTo("TYPE_VIEW_TEXT_TRAVERSED_AT_MOVEMENT_GRANULARITY") - assertThat(request.eventMap.get("event_package_name")).isEqualTo("some.loooong.package.name") - assertThat(request.eventMap.get("source_content_description")) - .isEqualTo("My Content Description") - assertThat(request.eventMap.get("source_text")).isEqualTo("My Source Text") - assertThat(request.eventMap.get("source_class_name")).isEqualTo("AwesomeClass") - assertThat(request.eventMap.get("event_text")).isEqualTo("Some text!") - assertThat(request.eventMap.get("event_timestamp_ms")).isEqualTo("1357924680") - assertThat(request.eventMap.get("movement_granularity")).isEqualTo("5") - assertThat(request.eventMap.get("from_index")).isEqualTo("6") - assertThat(request.eventMap.get("to_index")).isEqualTo("8") - assertThat(request.eventMap.get("view_id")).isEqualTo("this.big.old.view.id") - assertThat(request.eventMap.get("action")).isEqualTo("23") - // No error message - "" - } - - // Act. - forwarder.onAccessibilityEvent(event) - - // Assert. - // See `sendEventChecker` above. - } - - @Test - fun onAccessibilityEvent_sendingevent_grpcTimeout() { - // Arrange. - createForwarder( - logAccessibilityTree = false, - a11yTreePeriodMs = 0, - grpcHost = "amazing.host", - grpcPort = 4321, - ) - var nodeInfo = AccessibilityNodeInfo() - nodeInfo.setContentDescription("My Content Description") - nodeInfo.setText("My Source Text") - nodeInfo.setClassName("AwesomeClass") - var event = AccessibilityEvent() - event.setEventTime(1357924680) - event.setEventType(AccessibilityEvent.TYPE_VIEW_CLICKED) - event.getText().add("Some text!") - event.setPackageName("some.loooong.package.name") - shadowOf(event).setSourceNode(nodeInfo) - fakeA11yService.sendEventChecker = { _ -> - // Delay the request to prompt a timeout - Thread.sleep(1500L) - "" // Return no error. - } - - // Act. - forwarder.onAccessibilityEvent(event) - - // Run a second request to ensure that the channel gets rebuilt. - fakeA11yService.sendEventChecker = { _ -> "" } - forwarder.onAccessibilityEvent(event) - - // Assert. - // See `sendEventChecker` above. - } - - @Test - fun onAccessibilityEvent_sendingevent_grpcStatusException() { - // Arrange. - createForwarder(logAccessibilityTree = false, grpcHost = "amazing.host", grpcPort = 4321) - var nodeInfo = AccessibilityNodeInfo() - nodeInfo.setContentDescription("My Content Description") - nodeInfo.setText("My Source Text") - nodeInfo.setClassName("AwesomeClass") - var event = AccessibilityEvent() - event.setEventTime(1357924680) - event.setEventType(AccessibilityEvent.TYPE_VIEW_CLICKED) - event.getText().add("Some text!") - event.setPackageName("some.loooong.package.name") - shadowOf(event).setSourceNode(nodeInfo) - fakeA11yService.sendEventChecker = { _ -> throw StatusException(Status.UNAVAILABLE) } - - // Act. - forwarder.onAccessibilityEvent(event) - - // Run a second request to ensure that the channel gets rebuilt. - fakeA11yService.sendEventChecker = { _ -> "" } - forwarder.onAccessibilityEvent(event) - - // Assert. - // See `sendEventChecker` above. - } - - @Test - fun logAccessibilityTreeFalse_doesNotLogAccessibilityTree() { - // Arrange. - createForwarder(logAccessibilityTree = false, a11yTreePeriodMs = 10, grpcPort = 13579) - fakeA11yService.sendForestChecker = { _: AndroidAccessibilityForest -> - assertFalse(true) // This should not be called. - "" // This should be unreachable - } - - // Act. - Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. - - // Assert. - // See `sendForestChecker` above. - } - - @Test - fun grpcPortZero_doesNotSendTree() { - // Arrange. - createForwarder(logAccessibilityTree = true, a11yTreePeriodMs = 10, grpcPort = 0) - fakeA11yService.sendForestChecker = { _: AndroidAccessibilityForest -> - assertFalse(true) // This should not be called. - "" // This should be unreachable - } - - // Act. - Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. - - // Assert. - // See `sendForestChecker` above. - } - - @Test - fun grpcPortPositive_shouldSendTreeViaGrpc() { - // Arrange. - val window = AccessibilityWindowInfo() - shadowOf(window).setType(AccessibilityWindowInfo.TYPE_SYSTEM) - createForwarder( - logAccessibilityTree = true, - a11yTreePeriodMs = 10, - grpcPort = 1234, - a11yWindows = mutableListOf(window), - ) - fakeA11yService.sendForestChecker = { request: AndroidAccessibilityForest -> - // Check that we get only a single window. - assertThat(request.windowsList.size).isEqualTo(1) - // And that its type is what we set above. - assertThat(request.windowsList[0].windowType) - .isEqualTo(AndroidAccessibilityWindowInfo.WindowType.TYPE_SYSTEM) - // The error message - "Something went wrong!" - } - - // Act. - Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. - - // Assert. - // See `sendForestChecker` above. - } - - @Test - fun grpcPortPositiveAndHost_shouldSendTreeViaGrpc() { - // Arrange. - fakeA11yService.sendForestChecker = { request: AndroidAccessibilityForest -> - // Check that we get only a single window. - assertThat(request.windowsList.size).isEqualTo(1) - // And that its type is what we set above. - assertThat(request.windowsList[0].windowType) - .isEqualTo(AndroidAccessibilityWindowInfo.WindowType.TYPE_ACCESSIBILITY_OVERLAY) - "" // Return no error. - } - val window = AccessibilityWindowInfo() - shadowOf(window).setType(AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) - createForwarder( - logAccessibilityTree = true, - a11yTreePeriodMs = 500, - grpcHost = "amazing.host", - grpcPort = 4321, - a11yWindows = mutableListOf(window), - ) - - // Act. - Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. - - // Assert. - // See `sendForestChecker` above. - } - - @Test - fun sendingForest_grpcTimeout() { - // Arrange. - fakeA11yService.sendForestChecker = { _ -> - // Delay the request to prompt a timeout - Thread.sleep(1500L) - "" // Return no error. - } - val window = AccessibilityWindowInfo() - shadowOf(window).setType(AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) - createForwarder( - logAccessibilityTree = true, - a11yTreePeriodMs = 10, - grpcHost = "amazing.host", - grpcPort = 4321, - a11yWindows = mutableListOf(window), - ) - - // Act. - Thread.sleep(2000) // Sleep a bit to give time to trigger the tree logging function. - - // Run a second request to ensure that the channel gets rebuilt. - fakeA11yService.sendForestChecker = { _ -> "" } - Thread.sleep(2000) // Sleep a bit to give time to trigger the tree logging function. - - // Assert. - // See `sendForestChecker` above. - } - - @Test - fun sendingForest_grpcStatusException() { - // Arrange. - val window = AccessibilityWindowInfo() - shadowOf(window).setType(AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) - createForwarder( - logAccessibilityTree = true, - a11yTreePeriodMs = 10, - grpcHost = "amazing.host", - grpcPort = 4321, - a11yWindows = mutableListOf(window), - ) - fakeA11yService.sendForestChecker = { _ -> throw StatusException(Status.UNAVAILABLE) } - - // Act. - Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. - - // Run a second request to ensure that the channel gets rebuilt. - fakeA11yService.sendForestChecker = { _ -> "" } - Thread.sleep(1000) // Sleep a bit to give time to trigger the tree logging function. - - // Assert. - // See `sendForestChecker` above. - } -} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreator.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreator.kt deleted file mode 100644 index 87054118..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreator.kt +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2026 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.androidenv.accessibilityforwarder - -import android.graphics.Rect -import android.util.Log -import android.view.accessibility.AccessibilityNodeInfo -import android.view.accessibility.AccessibilityWindowInfo -import com.google.androidenv.accessibilityforwarder.AndroidAccessibilityWindowInfo.WindowType -import java.util.concurrent.ConcurrentHashMap -import java.util.stream.Collectors -import kotlin.collections.mutableListOf -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.runBlocking - -/** Helper methods for creating the android accessibility info extra. */ -class AccessibilityTreeCreator() { - - /** Creates an accessibility forest proto. */ - fun buildForest(windowInfos: List): AndroidAccessibilityForest { - val sourcesMap: ConcurrentHashMap = - ConcurrentHashMap() - val windows: List = - processWindowsAndBlock(windowInfos, sourcesMap) - return androidAccessibilityForest { this.windows += windows } - } - - private fun processWindowsAndBlock( - windowInfos: List, - sourcesMap: ConcurrentHashMap, - ): List { - val windows: List - runBlocking { windows = processWindows(windowInfos, sourcesMap) } - return windows - } - - private suspend fun processWindows( - windowInfos: List, - sourcesMap: ConcurrentHashMap, - ): List { - var windowInfoProtos = mutableListOf() - for (i in windowInfos.size - 1 downTo 0) { - val windowInfoProto = processWindow(windowInfos.get(i), sourcesMap) - windowInfoProto?.let { windowInfoProtos.add(windowInfoProto) } - } - return windowInfoProtos.toList() - } - - private suspend fun processWindow( - windowInfo: AccessibilityWindowInfo, - sources: ConcurrentHashMap, - ): AndroidAccessibilityWindowInfo? { - val bounds = Rect() - windowInfo.getBoundsInScreen(bounds) - val root: AccessibilityNodeInfo? = windowInfo.root - if (root == null) { - Log.i(TAG, "window root is null") - return androidAccessibilityWindowInfo { - this.tree = androidAccessibilityTree {} - this.isActive = windowInfo.isActive - this.id = windowInfo.id - this.layer = windowInfo.layer - this.isAccessibilityFocused = windowInfo.isAccessibilityFocused - this.isFocused = windowInfo.isFocused - this.boundsInScreen = convertToRectProto(bounds) - this.windowType = toWindowType(windowInfo.type) - } - } - val treeDeferred: Deferred - runBlocking { treeDeferred = async { processNodesInWindow(root, sources) } } - return androidAccessibilityWindowInfo { - this.tree = treeDeferred.await() - this.isActive = windowInfo.isActive - this.id = windowInfo.id - this.layer = windowInfo.layer - this.isAccessibilityFocused = windowInfo.isAccessibilityFocused - this.isFocused = windowInfo.isFocused - this.boundsInScreen = convertToRectProto(bounds) - this.windowType = toWindowType(windowInfo.type) - } - } - - private suspend fun processNodesInWindow( - root: AccessibilityNodeInfo, - sources: ConcurrentHashMap, - ): AndroidAccessibilityTree { - Log.d(TAG, "processNodesInWindow()") - val traversalQueue = ArrayDeque() - traversalQueue.add(ParentChildNodePair.builder().child(root).build()) - val uniqueIdsCache: UniqueIdsGenerator = UniqueIdsGenerator() - var currentDepth = 0 - val nodesDeferred = mutableListOf>() - val seenNodes: HashSet = HashSet() - seenNodes.add(root) - runBlocking { - while (!traversalQueue.isEmpty()) { - // Traverse the tree layer-by-layer. - // The first layer has only the root and depth 0. - // The second layer has all the root's children and depth 1. - for (nodesAtCurrentDepth in traversalQueue.size downTo 1) { - val nodePair: ParentChildNodePair = traversalQueue.removeFirst() - for (i in 0 until nodePair.child().childCount) { - val childNode: AccessibilityNodeInfo? = nodePair.child().getChild(i) - if (childNode != null && !seenNodes.contains(childNode)) { - traversalQueue.add( - ParentChildNodePair.builder().child(childNode).parent(nodePair.child()).build() - ) - seenNodes.add(childNode) - } - } - val thisDepth = currentDepth - var deferred = async { processNode(nodePair, sources, uniqueIdsCache, thisDepth) } - nodesDeferred.add(deferred) - } - currentDepth++ - } - } - return androidAccessibilityTree { this.nodes += nodesDeferred.awaitAll() } - } - - companion object { - private const val TAG = "AndroidRLTask" - } -} - -private fun processNode( - nodePair: ParentChildNodePair, - sourceBuilder: ConcurrentHashMap, - uniqueIdsCache: UniqueIdsGenerator, - nodeDepth: Int, -): AndroidAccessibilityNodeInfo { - val node: AccessibilityNodeInfo = nodePair.child() - val immutableNode: AndroidAccessibilityNodeInfo = - createAndroidAccessibilityNode( - node, - uniqueIdsCache.getUniqueId(node), - nodeDepth, - getChildUniqueIds(node, uniqueIdsCache), - ) - sourceBuilder.put(immutableNode, node) - return immutableNode -} - -private fun createAndroidAccessibilityNode( - node: AccessibilityNodeInfo, - nodeId: Int, - depth: Int, - childIds: List, -): AndroidAccessibilityNodeInfo { - val bounds = Rect() - node.getBoundsInScreen(bounds) - val actions = node.getActionList().stream().map(::createAction).collect(Collectors.toList()) - return androidAccessibilityNodeInfo { - this.actions += actions - this.boundsInScreen = convertToRectProto(bounds) - this.isCheckable = node.isCheckable - this.isChecked = node.isChecked - this.className = stringFromNullableCharSequence(node.getClassName()) - this.isClickable = node.isClickable - this.contentDescription = stringFromNullableCharSequence(node.getContentDescription()) - this.isEditable = node.isEditable - this.isEnabled = node.isEnabled - this.isFocusable = node.isFocusable - this.hintText = stringFromNullableCharSequence(node.getHintText()) - this.isLongClickable = node.isLongClickable - this.packageName = stringFromNullableCharSequence(node.getPackageName()) - this.isPassword = node.isPassword - this.isScrollable = node.isScrollable - this.isSelected = node.isSelected - this.text = stringFromNullableCharSequence(node.getText()) - this.textSelectionEnd = node.getTextSelectionEnd().toLong() - this.textSelectionStart = node.getTextSelectionStart().toLong() - this.viewIdResourceName = node.getViewIdResourceName() ?: "" - this.isVisibleToUser = node.isVisibleToUser - this.windowId = node.windowId - this.uniqueId = nodeId - this.childIds += childIds - this.drawingOrder = node.drawingOrder - this.tooltipText = stringFromNullableCharSequence(node.getTooltipText()) - this.depth = depth - } -} - -private fun createAction( - action: AccessibilityNodeInfo.AccessibilityAction -): AndroidAccessibilityAction = - AndroidAccessibilityAction.newBuilder() - .setId(action.id) - .setLabel(stringFromNullableCharSequence(action.label)) - .build() - -private fun getChildUniqueIds( - node: AccessibilityNodeInfo, - uniqueIdsCache: UniqueIdsGenerator, -): List { - val ids = mutableListOf() - for (childId in 0 until node.getChildCount()) { - val child: AccessibilityNodeInfo = node.getChild(childId) ?: continue - ids.add(uniqueIdsCache.getUniqueId(child)) - } - return ids.toList() -} - -fun stringFromNullableCharSequence(cs: CharSequence?): String = cs?.toString() ?: "" - -fun convertToRectProto(rect: Rect) = protoRect { - left = rect.left - top = rect.top - right = rect.right - bottom = rect.bottom -} - -private fun toWindowType(type: Int): WindowType = - when (type) { - AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY -> WindowType.TYPE_ACCESSIBILITY_OVERLAY - AccessibilityWindowInfo.TYPE_APPLICATION -> WindowType.TYPE_APPLICATION - AccessibilityWindowInfo.TYPE_INPUT_METHOD -> WindowType.TYPE_INPUT_METHOD - AccessibilityWindowInfo.TYPE_SYSTEM -> WindowType.TYPE_SYSTEM - AccessibilityWindowInfo.TYPE_SPLIT_SCREEN_DIVIDER -> WindowType.TYPE_SPLIT_SCREEN_DIVIDER - else -> WindowType.UNKNOWN_TYPE - } diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreatorTest.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreatorTest.kt deleted file mode 100644 index 0184b568..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AccessibilityTreeCreatorTest.kt +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2026 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.androidenv.accessibilityforwarder - -import android.view.accessibility.AccessibilityNodeInfo -import android.view.accessibility.AccessibilityWindowInfo -import kotlin.test.assertEquals -import org.junit.Test -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner -import org.robolectric.Shadows.shadowOf - -@RunWith(RobolectricTestRunner::class) -class AccessibilityTreeCreatorTest { - - @Test - fun buildForest_buildsAccessibilityForestCorrectly() { - val creator = AccessibilityTreeCreator() - - val forest = creator.buildForest(mutableListOf(createWindowInfo())) - - assertEquals(forest.windowsCount, 1) - assertEquals(forest.getWindows(0).tree.nodesCount, 3) - var rootNode: AndroidAccessibilityNodeInfo? = null - var checkableNode: AndroidAccessibilityNodeInfo? = null - val nodes = forest.getWindows(0).tree.nodesList - for (i in nodes.size - 1 downTo 0) { - if (nodes[i].text == "root node") { - rootNode = nodes[i] - } - if (nodes[i].isCheckable == true) { - checkableNode = nodes[i] - } - } - assertEquals(rootNode?.childIdsCount, 2) - assertEquals(checkableNode?.text, "Check box") - } - - @Test - fun buildForest_noRootInWindow_returnsEmptyTree() { - val creator = AccessibilityTreeCreator() - val windowInfo = AccessibilityWindowInfo.obtain() - shadowOf(windowInfo).setType(AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) - - val forest = creator.buildForest(mutableListOf(windowInfo)) - - assertEquals(0, forest.getWindows(0).tree.nodesList.size) - } - - private fun createAccessibilityNodeInfo(): AccessibilityNodeInfo { - val root = AccessibilityNodeInfo.obtain() - root.text = "root node" - root.isClickable = true - val accessibilityNodeInfo = AccessibilityNodeInfo.obtain() - accessibilityNodeInfo.viewIdResourceName = "test" - accessibilityNodeInfo.isClickable = true - accessibilityNodeInfo.isEditable = true - accessibilityNodeInfo.hintText = "Please enter your address" - shadowOf(root).addChild(accessibilityNodeInfo) - val anotherChildNode = AccessibilityNodeInfo.obtain() - anotherChildNode.isCheckable = true - anotherChildNode.text = "Check box" - shadowOf(root).addChild(anotherChildNode) - return root - } - - private fun createWindowInfo(): AccessibilityWindowInfo { - val windowInfo = AccessibilityWindowInfo.obtain() - shadowOf(windowInfo).setType(AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY) - shadowOf(windowInfo).setRoot(createAccessibilityNodeInfo()) - return windowInfo - } -} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest.xml b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest.xml deleted file mode 100644 index 07e4a860..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest_lite.xml b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest_lite.xml deleted file mode 100644 index 1d53cadc..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/AndroidManifest_lite.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiver.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiver.kt deleted file mode 100644 index 64530203..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiver.kt +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2026 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.androidenv.accessibilityforwarder - -import android.content.BroadcastReceiver -import android.content.Context -import android.content.Intent -import android.util.Log - -/** Broadcast receiver responsible for enabling or disabling flags. */ -class FlagsBroadcastReceiver() : BroadcastReceiver() { - - override fun onReceive(context: Context?, intent: Intent?) { - val action = intent?.action - Log.i(TAG, "Received broadcast intent with action: " + action) - when (action) { - ACTION_ENABLE_ACCESSIBILITY_TREE_LOGS -> { - Log.i(TAG, "Enabling Accessibility Tree logging.") - LogFlags.logAccessibilityTree = true - } - ACTION_DISABLE_ACCESSIBILITY_TREE_LOGS -> { - Log.i(TAG, "Disabling Accessibility Tree logging.") - LogFlags.logAccessibilityTree = false - } - ACTION_SET_GRPC -> { - // The Android Emulator uses 10.0.2.2 as a redirect to the workstation's IP. Most often the - // gRPC server will be running locally so it makes sense to use this as the default value. - // See https://developer.android.com/studio/run/emulator-networking#networkaddresses. - val host = intent.getStringExtra("host") ?: "10.0.2.2" - // The TCP port to connect. If <=0 gRPC is disabled. - val port = intent.getIntExtra("port", 0) - Log.i(TAG, "Setting gRPC endpoint to ${host}:${port}.") - LogFlags.grpcHost = host - LogFlags.grpcPort = port - } - else -> Log.w(TAG, "Unknown action: ${action}") - } - } - - companion object { - private const val TAG = "FlagsBroadcastReceiver" - private const val ACTION_ENABLE_ACCESSIBILITY_TREE_LOGS = - "accessibility_forwarder.intent.action.ENABLE_ACCESSIBILITY_TREE_LOGS" - private const val ACTION_DISABLE_ACCESSIBILITY_TREE_LOGS = - "accessibility_forwarder.intent.action.DISABLE_ACCESSIBILITY_TREE_LOGS" - private const val ACTION_SET_GRPC = "accessibility_forwarder.intent.action.SET_GRPC" - } -} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiverTest.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiverTest.kt deleted file mode 100644 index f1708f91..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/FlagsBroadcastReceiverTest.kt +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2026 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.androidenv.accessibilityforwarder - -import android.content.Intent -import com.google.common.truth.Truth.assertThat -import org.junit.Test -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner - -@RunWith(RobolectricTestRunner::class) -class FlagsBroadcastReceiverTest { - - @Test - fun onReceive_nullIntent_shouldNotLogAnything() { - // Arrange. - LogFlags.logAccessibilityTree = false - val receiver = FlagsBroadcastReceiver() - - // Act. - receiver.onReceive(context = null, intent = null) - - // Assert. - assertThat(LogFlags.logAccessibilityTree).isFalse() - } - - @Test - fun onReceive_nullIntent_actionShouldNotLogAnything() { - // Arrange. - LogFlags.logAccessibilityTree = false - val receiver = FlagsBroadcastReceiver() - val intent = Intent() - - // Act. - receiver.onReceive(context = null, intent = intent) - - // Assert. - assertThat(LogFlags.logAccessibilityTree).isFalse() - } - - @Test - fun onReceive_unknownIntent_actionShouldIssueWarning() { - // Arrange. - LogFlags.logAccessibilityTree = false - val receiver = FlagsBroadcastReceiver() - val intent = Intent("SOME_WEIRD_ACTION") - - // Act. - receiver.onReceive(context = null, intent = intent) - - // Assert. - assertThat(LogFlags.logAccessibilityTree).isFalse() - } - - @Test - fun onReceive_intentWithDisableAction_shouldDisableTreeLogging() { - // Arrange. - LogFlags.logAccessibilityTree = true - val receiver = FlagsBroadcastReceiver() - val intent = Intent("accessibility_forwarder.intent.action.DISABLE_ACCESSIBILITY_TREE_LOGS") - - // Act. - receiver.onReceive(context = null, intent = intent) - - // Assert. - assertThat(LogFlags.logAccessibilityTree).isFalse() - } - - @Test - fun onReceive_intentWithEnableAction_shouldEnableTreeLogging() { - // Arrange. - LogFlags.logAccessibilityTree = false - val receiver = FlagsBroadcastReceiver() - val intent = Intent("accessibility_forwarder.intent.action.ENABLE_ACCESSIBILITY_TREE_LOGS") - - // Act. - receiver.onReceive(context = null, intent = intent) - - // Assert. - assertThat(LogFlags.logAccessibilityTree).isTrue() - } - - @Test - fun onReceive_intentWithSetGrpcActionNoArgs_shouldDefaultToEmuIpAndPortZero() { - // Arrange. - LogFlags.grpcHost = "some_host" - LogFlags.grpcPort = 9999 - val receiver = FlagsBroadcastReceiver() - val intent = Intent("accessibility_forwarder.intent.action.SET_GRPC") - - // Act. - receiver.onReceive(context = null, intent = intent) - - // Assert. - assertThat(LogFlags.grpcHost).isEqualTo("10.0.2.2") - assertThat(LogFlags.grpcPort).isEqualTo(0) - } - - @Test - fun onReceive_intentWithSetGrpcActionWithHostNoPort_shouldDefaultPortToZero() { - // Arrange. - LogFlags.grpcHost = "some_host" - LogFlags.grpcPort = 9999 - val receiver = FlagsBroadcastReceiver() - val intent = - Intent("accessibility_forwarder.intent.action.SET_GRPC").apply { - putExtra("host", "awesome.server.ca") - } - - // Act. - receiver.onReceive(context = null, intent = intent) - - // Assert. - assertThat(LogFlags.grpcHost).isEqualTo("awesome.server.ca") - assertThat(LogFlags.grpcPort).isEqualTo(0) - } - - @Test - fun onReceive_intentWithSetGrpcActionWithPortNoHost_shouldDefaultHostToEmuIp() { - // Arrange. - LogFlags.grpcHost = "some_host" - LogFlags.grpcPort = 9999 - val receiver = FlagsBroadcastReceiver() - val intent = - Intent("accessibility_forwarder.intent.action.SET_GRPC").apply { putExtra("port", 54321) } - - // Act. - receiver.onReceive(context = null, intent = intent) - - // Assert. - assertThat(LogFlags.grpcHost).isEqualTo("10.0.2.2") - assertThat(LogFlags.grpcPort).isEqualTo(54321) - } - - @Test - fun onReceive_intentWithSetGrpcActionWithHostAndPort_shouldSetBoth() { - // Arrange. - LogFlags.grpcHost = "some_host" - LogFlags.grpcPort = 9999 - val receiver = FlagsBroadcastReceiver() - val intent = - Intent("accessibility_forwarder.intent.action.SET_GRPC").apply { - putExtra("host", "grpc.ca") - putExtra("port", 54321) - } - - // Act. - receiver.onReceive(context = null, intent = intent) - - // Assert. - assertThat(LogFlags.grpcHost).isEqualTo("grpc.ca") - assertThat(LogFlags.grpcPort).isEqualTo(54321) - } -} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/LogFlags.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/LogFlags.kt deleted file mode 100644 index 6482c8d2..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/LogFlags.kt +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2026 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.androidenv.accessibilityforwarder - -/** - * Controls global settings in AccessibilityForwarder. - * - * Please note that this class is not thread safe. - */ -object LogFlags { - // Whether to log the accessibility tree. - var logAccessibilityTree: Boolean = false - // How frequent to emit a11y trees (in milliseconds). - var a11yTreePeriodMs: Long = 100 - - // The gRPC server to connect to. (Only available if grpcPort>0). - var grpcHost: String = "" - // If >0 this represents the gRPC port number to connect to. - var grpcPort: Int = 0 -} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/ParentChildNodePair.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/ParentChildNodePair.kt deleted file mode 100644 index 7bc7ac6b..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/ParentChildNodePair.kt +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2026 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.androidenv.accessibilityforwarder - -import android.view.accessibility.AccessibilityNodeInfo -import com.google.auto.value.AutoValue - -/** Parent and child [AccessibilityNodeInfo] relationship. */ -@AutoValue -internal abstract class ParentChildNodePair { - abstract fun parent(): AccessibilityNodeInfo? - - abstract fun child(): AccessibilityNodeInfo - - /** [ParentChildNodePair] builder. */ - @AutoValue.Builder - abstract class Builder { - abstract fun parent(parent: AccessibilityNodeInfo?): Builder - - abstract fun child(child: AccessibilityNodeInfo): Builder - - abstract fun build(): ParentChildNodePair - } - - companion object { - @JvmStatic fun builder(): Builder = AutoValue_ParentChildNodePair.Builder() - } -} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/UniqueIdsGenerator.kt b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/UniqueIdsGenerator.kt deleted file mode 100644 index ee5609dd..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/UniqueIdsGenerator.kt +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2026 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.androidenv.accessibilityforwarder - -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicInteger -import java.util.function.Function - -/** Thread-safe helper class for assigning a unique ID to an object. */ -internal class UniqueIdsGenerator { - private val nextId = AtomicInteger(0) - private val uniqueIdsByNode = ConcurrentHashMap() - - fun getUniqueId(a: A): Int { - return uniqueIdsByNode.computeIfAbsent(a, Function { _: A -> nextId.getAndIncrement() })!! - } -} diff --git a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/res/xml/accessibility_forwarder_service.xml b/android_env/apps/java/com/google/androidenv/accessibilityforwarder/res/xml/accessibility_forwarder_service.xml deleted file mode 100644 index 37643f8d..00000000 --- a/android_env/apps/java/com/google/androidenv/accessibilityforwarder/res/xml/accessibility_forwarder_service.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - diff --git a/android_env/env_interface.py b/android_env/env_interface.py index ffd58a9d..5c850098 100644 --- a/android_env/env_interface.py +++ b/android_env/env_interface.py @@ -23,7 +23,6 @@ from typing import Any from android_env.proto import adb_pb2 -from android_env.proto import state_pb2 import dm_env from dm_env import specs import numpy as np @@ -78,33 +77,3 @@ def execute_adb_call(self, call: adb_pb2.AdbRequest) -> adb_pb2.AdbResponse: """Executes `call` and returns its response.""" return adb_pb2.AdbResponse() - - def load_state( - self, request: state_pb2.LoadStateRequest - ) -> state_pb2.LoadStateResponse: - """Loads a state. - - Args: - request: A `LoadStateRequest` containing any parameters necessary to - specify how/what state to load. - - Returns: - A `LoadStateResponse` containing the status, error message (if - applicable), and any other relevant information. - """ - raise NotImplementedError('This environment does not support loading state') - - def save_state( - self, request: state_pb2.SaveStateRequest - ) -> state_pb2.SaveStateResponse: - """Saves a state. - - Args: - request: A `SaveStateRequest` containing any parameters necessary to - specify how/what state to save. - - Returns: - A `SaveStateResponse` containing the status, error message (if - applicable), and any other relevant information. - """ - raise NotImplementedError('This environment does not support saving state') diff --git a/android_env/proto/BUILD.bazel b/android_env/proto/BUILD.bazel index 4fecc00c..cc3853bc 100644 --- a/android_env/proto/BUILD.bazel +++ b/android_env/proto/BUILD.bazel @@ -1,5 +1,5 @@ load("@rules_proto//proto:defs.bzl", "proto_library") -load("//:py_grpc_library.bzl", "py_proto_library") +load("//:py_proto_library.bzl", "py_proto_library") package(default_visibility = ["//visibility:public"]) @@ -26,16 +26,6 @@ py_proto_library( deps = [":adb_py_pb2"], ) -proto_library( - name = "state_proto", - srcs = ["state.proto"], -) - -py_proto_library( - name = "state_py_pb2", - srcs = ["state.proto"], -) - proto_library( name = "device_env_service_proto", srcs = ["device_env_service.proto"], diff --git a/android_env/proto/a11y/BUILD.bazel b/android_env/proto/a11y/BUILD.bazel index 1cb8a821..46c76c1a 100644 --- a/android_env/proto/a11y/BUILD.bazel +++ b/android_env/proto/a11y/BUILD.bazel @@ -1,5 +1,5 @@ load("@rules_proto//proto:defs.bzl", "proto_library") -load("//:py_grpc_library.bzl", "py_grpc_library", "py_proto_library") +load("//:py_proto_library.bzl", "py_proto_library") package(default_visibility = ["//visibility:public"]) @@ -26,21 +26,6 @@ py_proto_library( deps = [":android_accessibility_forest_py_pb2"], ) -py_grpc_library( - name = "a11y_py_pb2_grpc", - srcs = ["a11y.proto"], - proto_srcs = [ - "android_accessibility_action.proto", - "android_accessibility_forest.proto", - "android_accessibility_node_info.proto", - "android_accessibility_node_info_clickable_span.proto", - "android_accessibility_tree.proto", - "android_accessibility_window_info.proto", - "rect.proto", - ], - deps = [":a11y_py_pb2"], -) - proto_library( name = "android_accessibility_action_proto", srcs = ["android_accessibility_action.proto"], diff --git a/android_env/proto/state.proto b/android_env/proto/state.proto deleted file mode 100644 index 8dcf4037..00000000 --- a/android_env/proto/state.proto +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2026 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package android_env; - -option java_multiple_files = true; - -message SaveStateRequest { - map args = 1; -} - -message LoadStateRequest { - map args = 1; -} - -message SaveStateResponse { - enum Status { - // Reserved value for unset statuses. - UNDEFINED = 0; - // Returned when everything goes well. - OK = 1; - // Returned when something internal did not work as expected. - ERROR = 2; - } - Status status = 1; - // `error_message` is only populated in case of errors. - string error_message = 2; - - // Any additional info returned during the request; e.g., file paths or sizes. - map additional_info = 3; -} - -message LoadStateResponse { - enum Status { - // Reserved value for unset statuses. - UNDEFINED = 0; - // Returned when everything goes well. - OK = 1; - // Returned when there is no state to load. - NOT_FOUND = 2; - // Returned when something internal did not work as expected. - ERROR = 3; - } - Status status = 1; - // `error_message` is only populated in case of errors. - string error_message = 2; - - // Any additional info returned during the request; e.g., file paths or sizes. - map additional_info = 3; -} \ No newline at end of file diff --git a/android_env/wrappers/BUILD.bazel b/android_env/wrappers/BUILD.bazel index 7bef5002..a7160412 100644 --- a/android_env/wrappers/BUILD.bazel +++ b/android_env/wrappers/BUILD.bazel @@ -2,51 +2,12 @@ load("@rules_python//python:defs.bzl", "py_library", "py_test") package(default_visibility = ["//visibility:public"]) -py_library( - name = "a11y_grpc_wrapper", - srcs = ["a11y_grpc_wrapper.py"], - deps = [ - ":base_wrapper", - "//android_env:env_interface", - "//android_env/components:action_type", - "//android_env/proto:adb_py_pb2", - "//android_env/proto/a11y:a11y_py_pb2_grpc", - "//android_env/wrappers/a11y:a11y_events", - "//android_env/wrappers/a11y:a11y_forests", - "//android_env/wrappers/a11y:a11y_servicer", - "@pip//absl_py", - "@pip//dm_env", - "@pip//grpcio", - "@pip//numpy", - "@pip//portpicker", - ], -) - -py_test( - name = "a11y_grpc_wrapper_test", - srcs = ["a11y_grpc_wrapper_test.py"], - legacy_create_init = False, - deps = [ - ":a11y_grpc_wrapper", - "//android_env:env_interface", - "//android_env/proto:adb_py_pb2", - "//android_env/proto/a11y:a11y_py_pb2_grpc", - "//android_env/proto/a11y:android_accessibility_forest_py_pb2", - "@pip//absl_py", - "@pip//dm_env", - "@pip//grpcio", - "@pip//numpy", - "@pip//portpicker", - ], -) - py_library( name = "base_wrapper", srcs = ["base_wrapper.py"], deps = [ "//android_env:env_interface", "//android_env/proto:adb_py_pb2", - "//android_env/proto:state_py_pb2", "@pip//absl_py", "@pip//dm_env", "@pip//numpy", @@ -60,7 +21,6 @@ py_test( deps = [ ":base_wrapper", "//android_env:env_interface", - "//android_env/proto:state_py_pb2", "@pip//absl_py", ], ) diff --git a/android_env/wrappers/a11y/BUILD.bazel b/android_env/wrappers/a11y/BUILD.bazel deleted file mode 100644 index e438bd04..00000000 --- a/android_env/wrappers/a11y/BUILD.bazel +++ /dev/null @@ -1,74 +0,0 @@ -load("@rules_python//python:defs.bzl", "py_library", "py_test") - -package(default_visibility = ["//android_env/wrappers:__subpackages__"]) - -py_library( - name = "a11y_events", - srcs = ["a11y_events.py"], - deps = [ - "//android_env/proto/a11y:a11y_py_pb2_grpc", # Redirected from a11y_py_pb2 - "@pip//absl_py", - "@pip//numpy", - "@pip//protobuf", # For any_py_pb2 - ], -) - -py_test( - name = "a11y_events_test", - srcs = ["a11y_events_test.py"], - legacy_create_init = False, - deps = [ - ":a11y_events", - "//android_env/proto/a11y:a11y_py_pb2_grpc", - "@pip//absl_py", - "@pip//numpy", - "@pip//protobuf", - ], -) - -py_library( - name = "a11y_forests", - srcs = ["a11y_forests.py"], - deps = [ - "//android_env/proto/a11y:android_accessibility_forest_py_pb2", - "@pip//numpy", - "@pip//protobuf", - ], -) - -py_test( - name = "a11y_forests_test", - srcs = ["a11y_forests_test.py"], - legacy_create_init = False, - deps = [ - ":a11y_forests", - "//android_env/proto/a11y:android_accessibility_forest_py_pb2", - "@pip//absl_py", - "@pip//numpy", - "@pip//protobuf", - ], -) - -py_library( - name = "a11y_servicer", - srcs = ["a11y_servicer.py"], - deps = [ - "//android_env/proto/a11y:a11y_py_pb2_grpc", - "//android_env/proto/a11y:android_accessibility_forest_py_pb2", - "@pip//absl_py", - "@pip//grpcio", - ], -) - -py_test( - name = "a11y_servicer_test", - srcs = ["a11y_servicer_test.py"], - legacy_create_init = False, - deps = [ - ":a11y_servicer", - "//android_env/proto/a11y:a11y_py_pb2_grpc", - "//android_env/proto/a11y:android_accessibility_forest_py_pb2", - "@pip//absl_py", - "@pip//grpcio", - ], -) diff --git a/android_env/wrappers/a11y/__init__.py b/android_env/wrappers/a11y/__init__.py deleted file mode 100644 index 0256ac69..00000000 --- a/android_env/wrappers/a11y/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# coding=utf-8 -# Copyright 2026 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - diff --git a/android_env/wrappers/a11y/a11y_events.py b/android_env/wrappers/a11y/a11y_events.py deleted file mode 100644 index 9a72f814..00000000 --- a/android_env/wrappers/a11y/a11y_events.py +++ /dev/null @@ -1,117 +0,0 @@ -# coding=utf-8 -# Copyright 2026 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tools for accessing accessibility events.""" - -from collections.abc import Mapping -from typing import Any - -from absl import logging -from android_env.proto.a11y import a11y_pb2 -import numpy as np - -from google.protobuf import any_pb2 - - -_A11Y_EVENT_KEY = 'full_event' - - -def package_events_to_task_extras( - events: list[a11y_pb2.EventRequest], -) -> Mapping[str, np.ndarray]: - if not events: - return {} - return {_A11Y_EVENT_KEY: np.array(events, dtype=object)} - - -def extract_events_from_task_extras( - task_extras: Mapping[str, Any] | None = None, -) -> list[Mapping[str, str]]: - """Inspects task_extras and extracts all accessibility events detected. - - Args: - task_extras: Task extras forwarded by AndroidEnv. If 'full_event' is not a - key in task_extras, then this function returns an empty string. Otherwise, - full_event is expected to be list to be a numpy array with one dimension, - and contains a list of dictionary describing accessibility events that are - present in the given task extras. e.g. 'event_type: - TYPE_WINDOW_CONTENT_CHANGED // event_package_name: - com.google.android.deskclock // source_class_name: - android.widget.ImageView'. - - Returns: - List of all events detected - """ - if task_extras is None or _A11Y_EVENT_KEY not in task_extras: - return [] - - if ( - not isinstance(task_extras[_A11Y_EVENT_KEY], np.ndarray) - or task_extras[_A11Y_EVENT_KEY].ndim != 1 - ): - raise ValueError( - f'{_A11Y_EVENT_KEY} task extra should be a numpy array with one' - ' dimension.' - ) - - if task_extras[_A11Y_EVENT_KEY].size == 0: - return [] - - events = [] - for e in task_extras[_A11Y_EVENT_KEY]: - if isinstance(e, a11y_pb2.EventRequest): - events.append(dict(e.event)) - elif isinstance(e, dict): - events.append(e) - logging.warning( - 'The event should come only from the a11y_grpc_wrapper. ' - 'Please verify that the upacking operation has not been ' - 'called twice. See here for full task_extras: %s', - task_extras, - ) - elif isinstance(e, any_pb2.Any): - ev = a11y_pb2.EventRequest() - new_any = any_pb2.Any() - new_any.CopyFrom(e) - new_any.Unpack(ev) - events.append(dict(ev.event)) - - else: - raise TypeError( - f'Unexpected event type: {type(e)}. See here for full ' - f'task_extras: {task_extras}.' - ) - - return events - - -def keep_latest_event_only(task_extras: dict[str, Any]): - """Removes all a11y events except the last one observed.""" - if task_extras is None or 'full_event' not in task_extras: - return - - if ( - not isinstance(task_extras[_A11Y_EVENT_KEY], np.ndarray) - or task_extras[_A11Y_EVENT_KEY].ndim != 1 - ): - raise ValueError( - f'{_A11Y_EVENT_KEY} task extra should be a numpy array with one' - ' dimension.' - ) - - if task_extras[_A11Y_EVENT_KEY].size == 0: - return [] - - task_extras[_A11Y_EVENT_KEY] = task_extras[_A11Y_EVENT_KEY][-1:] diff --git a/android_env/wrappers/a11y/a11y_events_test.py b/android_env/wrappers/a11y/a11y_events_test.py deleted file mode 100644 index 193f6eb0..00000000 --- a/android_env/wrappers/a11y/a11y_events_test.py +++ /dev/null @@ -1,173 +0,0 @@ -# coding=utf-8 -# Copyright 2026 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for a11y_events.""" - -from absl.testing import absltest -from absl.testing import parameterized -from android_env.proto.a11y import a11y_pb2 -from android_env.wrappers.a11y import a11y_events -import numpy as np - -from google.protobuf import any_pb2 - - -def _event_request(d: dict[str, str]) -> a11y_pb2.EventRequest: - event_request = a11y_pb2.EventRequest() - for k, v in d.items(): - event_request.event[k] = v - return event_request - - -def _event_request_as_any(d: dict[str, str]) -> any_pb2.Any: - event_request = _event_request(d) - response = any_pb2.Any() - response.Pack(event_request) - return response - - -class A11yEventsTest(parameterized.TestCase): - - @parameterized.parameters( - dict(task_extras={}), - dict( - task_extras={'no_full_event': [{'1': '1'}, {'2': '2'}, {'3': '3'}]}, - ), - dict( - task_extras={'full_event': np.array([])}, - ), - dict( - task_extras={}, - ), - ) - def test_no_events_in_task_extras(self, task_extras): - events = a11y_events.extract_events_from_task_extras(task_extras) - self.assertEmpty(events) - - @parameterized.parameters( - dict( - task_extras={'full_event': [{'1': '1'}, {'2': '2'}]}, - expected_events=[{'1': '1'}, {'2': '2'}], - ), - dict( - task_extras={'full_event': [{}]}, - expected_events=[{}], - ), - dict( - task_extras={ - 'full_event_wrong_key': [1, 2, 3], - 'full_event': [{'1': '1'}, {'2': '2'}, {'3': '3'}], - }, - expected_events=[{'1': '1'}, {'2': '2'}, {'3': '3'}], - ), - ) - def test_task_extras(self, task_extras, expected_events): - event_requests = [_event_request(e) for e in task_extras['full_event']] - task_extras['full_event'] = np.array(event_requests, dtype=object) - events = a11y_events.extract_events_from_task_extras(task_extras) - self.assertEqual(len(events), len(expected_events)) - for i, event in enumerate(expected_events): - self.assertEqual(len(event), len(expected_events[i])) - for k, v in event.items(): - self.assertIn(k, expected_events[i]) - self.assertEqual(v, expected_events[i][k]) - - def test_events_key_has_dict_event_requrests(self): - event_requests = [ - _event_request({'1': '1'}), - {'2': '2'}, - _event_request({'3': '3'}), - ] - expected_events = [ - {'1': '1'}, - {'2': '2'}, - {'3': '3'}, - ] - task_extras = {'full_event': np.array(event_requests, dtype=object)} - events = a11y_events.extract_events_from_task_extras(task_extras) - self.assertEqual(len(events), len(expected_events)) - for i, event in enumerate(expected_events): - self.assertEqual(len(event), len(expected_events[i])) - for k, v in event.items(): - self.assertIn(k, expected_events[i]) - self.assertEqual(v, expected_events[i][k]) - - def test_events_key_has__event_requrests_packed_as_any(self): - event_requests = [ - _event_request_as_any({'1': '1'}), - {'2': '2'}, - _event_request_as_any({'3': '3'}), - ] - expected_events = [ - {'1': '1'}, - {'2': '2'}, - {'3': '3'}, - ] - task_extras = {'full_event': np.array(event_requests, dtype=object)} - events = a11y_events.extract_events_from_task_extras(task_extras) - self.assertEqual(len(events), len(expected_events)) - for i, event in enumerate(expected_events): - self.assertEqual(len(event), len(expected_events[i])) - for k, v in event.items(): - self.assertIn(k, expected_events[i]) - self.assertEqual(v, expected_events[i][k]) - - def test_events_key_has_non_event_requrests(self): - event_requests = [ - _event_request({'1': '1'}), - 3, # Not an even and not a dict. - _event_request({'3': '3'}), - ] - task_extras = {'full_event': np.array(event_requests, dtype=object)} - with self.assertRaises(TypeError): - _ = a11y_events.extract_events_from_task_extras(task_extras) - - @parameterized.parameters( - dict(task_extras={}, expected_extras={}), - dict( - task_extras={ - 'no_full_event': 42, - }, - expected_extras={ - 'no_full_event': 42, - }, - ), - dict( - task_extras={'full_event': np.array([1, 2]), 'no_full_event': 43}, - expected_extras={'full_event': np.array([2]), 'no_full_event': 43}, - ), - dict( - task_extras={'full_event': np.array([1, 2, 3])}, - expected_extras={'full_event': np.array([3])}, - ), - dict( - task_extras={'full_event': np.array([]), 'no_full_event': 44}, - expected_extras={'full_event': np.array([]), 'no_full_event': 44}, - ), - ) - def test_keep_latest_only(self, task_extras, expected_extras): - a11y_events.keep_latest_event_only(task_extras) - self.assertEqual(len(task_extras), len(expected_extras)) - for k, v in task_extras.items(): - self.assertIn(k, expected_extras) - if k == 'full_event': - np.testing.assert_array_equal(v, expected_extras['full_event']) - else: - self.assertEqual(v, expected_extras[k]) - pass - - -if __name__ == '__main__': - absltest.main() diff --git a/android_env/wrappers/a11y/a11y_forests.py b/android_env/wrappers/a11y/a11y_forests.py deleted file mode 100644 index 3857de15..00000000 --- a/android_env/wrappers/a11y/a11y_forests.py +++ /dev/null @@ -1,127 +0,0 @@ -# coding=utf-8 -# Copyright 2026 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tools for accessing accessibility events.""" - -from collections.abc import Mapping -from typing import Any - -from android_env.proto.a11y import android_accessibility_forest_pb2 -import numpy as np - -from google.protobuf import any_pb2 - - -_A11Y_FORESTS_KEY = 'accessibility_tree' - - -def package_forests_to_task_extras( - forests: list[android_accessibility_forest_pb2.AndroidAccessibilityForest], -) -> Mapping[str, np.ndarray]: - if not forests: - return {} - return {_A11Y_FORESTS_KEY: np.array(forests, dtype=object)} - - -def task_extras_has_forests(task_extras: Mapping[str, Any]) -> bool: - """Checks that the task_extras has any a11y forest information.""" - if _A11Y_FORESTS_KEY not in task_extras: - return False - - payload = task_extras[_A11Y_FORESTS_KEY] - if not isinstance(payload, np.ndarray) or payload.ndim != 1: - raise ValueError( - f'{_A11Y_FORESTS_KEY} task extra should be a numpy array with one' - f' dimension. payload: {payload}' - ) - - if payload.size == 0: - return False - - if any(isinstance(f, any_pb2.Any) for f in payload): - # Forests were packed as Any. - return True - - return any( - isinstance(f, android_accessibility_forest_pb2.AndroidAccessibilityForest) - for f in payload - ) - - -def convert_to_forest( - forest: android_accessibility_forest_pb2.AndroidAccessibilityForest - | any_pb2.Any - | None, -) -> android_accessibility_forest_pb2.AndroidAccessibilityForest | None: - """Takes an object and attempts to convert it to a forest.""" - if forest is None: - return None - - if isinstance(forest, any_pb2.Any): - output = android_accessibility_forest_pb2.AndroidAccessibilityForest() - new_any = any_pb2.Any() - new_any.CopyFrom(forest) - new_any.Unpack(output) - return output - elif isinstance( - forest, android_accessibility_forest_pb2.AndroidAccessibilityForest - ): - return forest - else: - return None - - -def extract_forests_from_task_extras( - task_extras: Mapping[str, Any] | None = None, -) -> list[android_accessibility_forest_pb2.AndroidAccessibilityForest]: - """Inspects task_extras and extracts all accessibility forests detected. - - Args: - task_extras: Task extras forwarded by AndroidEnv. If 'full_event' is not a - key in task_extras, then this function returns an empty string. Otherwise, - full_event is expected to be list to be a numpy array with one dimension, - and contains a list of dictionary describing accessibility forests that - are present in the given task extras. - - Returns: - List of all forests detected - """ - if task_extras is None or not task_extras_has_forests(task_extras): - return [] - - forests = [] - for f in task_extras[_A11Y_FORESTS_KEY]: - f = convert_to_forest(f) - if f is not None: - forests.append(f) - return forests - - -def keep_latest_forest_only(task_extras: dict[str, Any]): - """Removes all a11y forests except the last one observed.""" - if _A11Y_FORESTS_KEY not in task_extras.keys(): - return - - payload = task_extras[_A11Y_FORESTS_KEY] - if not isinstance(payload, np.ndarray) or payload.ndim != 1: - raise ValueError( - f'{_A11Y_FORESTS_KEY} task extra should be a numpy array with one' - f' dimension. payload: {payload}' - ) - - if payload.size == 0: - return - - task_extras[_A11Y_FORESTS_KEY] = payload[-1:] diff --git a/android_env/wrappers/a11y/a11y_forests_test.py b/android_env/wrappers/a11y/a11y_forests_test.py deleted file mode 100644 index e4457759..00000000 --- a/android_env/wrappers/a11y/a11y_forests_test.py +++ /dev/null @@ -1,237 +0,0 @@ -# coding=utf-8 -# Copyright 2026 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for a11y_forests.""" - -from absl.testing import absltest -from absl.testing import parameterized -from android_env.proto.a11y import android_accessibility_forest_pb2 -from android_env.wrappers.a11y import a11y_forests -import numpy as np - -from google.protobuf import any_pb2 - - -def _pack_any(proto_message) -> any_pb2.Any: - response = any_pb2.Any() - response.Pack(proto_message) - return response - - -def _empty_forest() -> ( - android_accessibility_forest_pb2.AndroidAccessibilityForest -): - return android_accessibility_forest_pb2.AndroidAccessibilityForest() - - -def _one_empty_window_forest() -> ( - android_accessibility_forest_pb2.AndroidAccessibilityForest -): - forest = android_accessibility_forest_pb2.AndroidAccessibilityForest() - forest.windows.add() - return forest - - -def _two_window_forest() -> ( - android_accessibility_forest_pb2.AndroidAccessibilityForest -): - forest = android_accessibility_forest_pb2.AndroidAccessibilityForest() - window = forest.windows.add() - window.tree.nodes.add( - class_name='foo', is_clickable=True, hint_text='Foo hint' - ) - forest.windows.add() - return forest - - -class A11YForestsTest(parameterized.TestCase): - - @parameterized.parameters( - dict(task_extras={}, expected_forests=[], convert_to_np=[]), - dict( - task_extras={'accessibility_tree': []}, - convert_to_np=['accessibility_tree'], - expected_forests=[], - ), - dict( - task_extras={ - 'not_accessibility_tree': [ - _empty_forest(), - _one_empty_window_forest(), - _two_window_forest(), - ], - }, - convert_to_np=['not_accessibility_tree'], - expected_forests=[], - ), - dict( - task_extras={ - 'accessibility_tree': [ - _empty_forest(), - {'not_a_forest_key': 'nor_a_forest_value'}, - _two_window_forest(), - ] - }, - convert_to_np=['accessibility_tree'], - expected_forests=[_empty_forest(), _two_window_forest()], - ), - dict( - task_extras={ - 'accessibility_tree': [ - {'not_a_forest_key': 'nor_a_forest_value'}, - 3, - 4, - {'not_a_forest_key': _empty_forest()}, - ], - }, - convert_to_np=['accessibility_tree'], - expected_forests=[], - ), - dict( - task_extras={'accessibility_tree': []}, - convert_to_np=['accessibility_tree'], - expected_forests=[], - ), - dict( - task_extras={ - 'accessibility_tree_wrong_key': [1, 2, 3], - 'accessibility_tree': [ - _empty_forest(), - None, - None, - _one_empty_window_forest(), - _two_window_forest(), - ], - }, - convert_to_np=['accessibility_tree', 'accessibility_tree_wrong_key'], - expected_forests=[ - _empty_forest(), - _one_empty_window_forest(), - _two_window_forest(), - ], - ), - dict( - task_extras={ - 'accessibility_tree_wrong_key': [1, 2, 3], - 'accessibility_tree': [ - None, - _pack_any(_empty_forest()), - _pack_any(_one_empty_window_forest()), - _pack_any(_two_window_forest()), - ], - }, - convert_to_np=['accessibility_tree', 'accessibility_tree_wrong_key'], - expected_forests=[ - _empty_forest(), - _one_empty_window_forest(), - _two_window_forest(), - ], - ), - dict( - task_extras={ - 'accessibility_tree': [ - _pack_any(_empty_forest()), - {'not_a_forest_key': 'nor_a_forest_value'}, - None, - _two_window_forest(), - None, - ] - }, - convert_to_np=['accessibility_tree'], - expected_forests=[_empty_forest(), _two_window_forest()], - ), - ) - def test_task_extras(self, task_extras, expected_forests, convert_to_np): - for k in convert_to_np: - if task_extras[k]: - task_extras[k] = np.array(task_extras[k], dtype=object) - else: - task_extras[k] = np.array([]) - forests = a11y_forests.extract_forests_from_task_extras(task_extras) - self.assertEqual(len(forests), len(expected_forests)) - for idx, f in enumerate(forests): - self.assertEqual(f, expected_forests[idx]) - - @parameterized.parameters( - dict(task_extras={}, expected_extras={}), - dict( - task_extras={ - 'no_accessibility_tree': 42, - }, - expected_extras={ - 'no_accessibility_tree': 42, - }, - ), - dict( - task_extras={'accessibility_tree': []}, - expected_extras={'accessibility_tree': []}, - ), - dict( - task_extras={ - 'accessibility_tree': [ - _empty_forest(), - _one_empty_window_forest(), - ], - 'no_accessibility_tree': 43, - }, - expected_extras={ - 'accessibility_tree': [_one_empty_window_forest()], - 'no_accessibility_tree': 43, - }, - ), - dict( - task_extras={ - 'accessibility_tree': [ - _empty_forest(), - _one_empty_window_forest(), - _two_window_forest(), - ] - }, - expected_extras={'accessibility_tree': [_two_window_forest()]}, - ), - dict( - task_extras={ - 'accessibility_tree': [], - 'no_accessibility_tree': 44, - }, - expected_extras={ - 'accessibility_tree': [], - 'no_accessibility_tree': 44, - }, - ), - ) - def test_keep_latest_only(self, task_extras, expected_extras): - if 'accessibility_tree' in task_extras: - if task_extras['accessibility_tree']: - task_extras['accessibility_tree'] = np.array( - task_extras['accessibility_tree'], dtype=object - ) - else: - task_extras['accessibility_tree'] = np.array([]) - - a11y_forests.keep_latest_forest_only(task_extras) - self.assertSameElements(task_extras.keys(), expected_extras.keys()) - for k in task_extras.keys(): - if k == 'accessibility_tree': - self.assertEqual(len(task_extras[k]), len(expected_extras[k])) - for idx, f in enumerate(task_extras[k]): - self.assertEqual(f, expected_extras[k][idx]) - else: - self.assertEqual(task_extras[k], expected_extras[k]) - pass - - -if __name__ == '__main__': - absltest.main() diff --git a/android_env/wrappers/a11y/a11y_servicer.py b/android_env/wrappers/a11y/a11y_servicer.py deleted file mode 100644 index 6e9378fd..00000000 --- a/android_env/wrappers/a11y/a11y_servicer.py +++ /dev/null @@ -1,199 +0,0 @@ -# coding=utf-8 -# Copyright 2026 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Accessibility Servicer implementation.""" - -import asyncio -from collections.abc import AsyncIterator, Generator, Iterable -import threading - -from absl import logging -from android_env.proto.a11y import a11y_pb2 -from android_env.proto.a11y import a11y_pb2_grpc -from android_env.proto.a11y import android_accessibility_forest_pb2 -import grpc - - -class A11yServicer(a11y_pb2_grpc.A11yServiceServicer): - """Services the A11yService requests.""" - - def __init__(self, latest_forest_only: bool = False): - self._received_forests: list[ - android_accessibility_forest_pb2.AndroidAccessibilityForest - ] = [] - self._received_events: list[a11y_pb2.EventRequest] = [] - self._lock_forests = threading.Lock() - self._lock_events = threading.Lock() - self._latest_forest_only = latest_forest_only - self._paused = True - - # A11y Forest bookkeeping. - self._get_forest = asyncio.Event() # Whether to request a forest. - self._forest_ready = asyncio.Event() # Whether the forest is ready. - self._latest_forest: ( - android_accessibility_forest_pb2.AndroidAccessibilityForest | None - ) = None - - def SendForest( - self, - request: android_accessibility_forest_pb2.AndroidAccessibilityForest, - context: grpc.ServicerContext, - ) -> a11y_pb2.ForestResponse: - self._process_forest(request) - return a11y_pb2.ForestResponse() - - def SendEvent( - self, - request: a11y_pb2.EventRequest, - context: grpc.ServicerContext, - ) -> a11y_pb2.EventResponse: - self._process_event(request) - return a11y_pb2.EventResponse() - - async def Bidi( - self, - request_iterator: AsyncIterator[a11y_pb2.ClientToServer], - context: grpc.aio.ServicerContext, - ) -> AsyncIterator[a11y_pb2.ServerToClient]: - """Processes incoming ClientToServer requests.""" - - logging.info('Starting A11yServicer.Bidi()') - - # Send a dummy message to unblock clients in their loop. - yield a11y_pb2.ServerToClient() - - # This block defines two coroutines: - # - # * `read_client_requests()` - # * `check_forest()` - # - # They cooperate with each other and both populate a queue `q` which is - # consumed in a loop below, which actually yields requests which are sent to - # the client. The processing finishes when the clients "closes" the - # connection, which causes `read_client_requests()` to put a special value, - # `STOP_ITERATION`, in the queue. - - # Queue for communicating from coroutines to `Bidi()`. - q = asyncio.Queue() - - should_run = True - - async def read_client_requests(): - """Coroutine for reading client requests.""" - - nonlocal should_run - async for request in request_iterator: - field_name = request.WhichOneof('payload') - match field_name: - case 'event': - self._process_event(request.event) - case 'forest': - self._latest_forest = request.forest - self._forest_ready.set() - self._get_forest.clear() # Reset the `Event`. - case _: - logging.error('Unknown field %r', field_name) - await q.put(a11y_pb2.ServerToClient()) - - # Send a special value to stop processing this `Bidi` connection. - await q.put('STOP_ITERATION') - should_run = False - - async def check_forest(): - """Coroutine for sending "get forest" requests.""" - - nonlocal should_run - while should_run: - await self._get_forest.wait() - await q.put(a11y_pb2.ServerToClient(get_forest={})) - - tasks = asyncio.gather(read_client_requests(), check_forest()) - - while should_run: - v = await q.get() - if v == 'STOP_ITERATION': - break - else: - yield v - - await tasks - - logging.info('Finishing A11yServicer.Bidi()') - - async def get_forest( - self, - ) -> android_accessibility_forest_pb2.AndroidAccessibilityForest | None: - """Issues a request to get the a11y forest from the client.""" - - self._get_forest.set() # Unblocks coroutine to send a request. - await self._forest_ready.wait() # Wait for forest to be ready. - self._forest_ready.clear() # Reset the `Event`. - return self._latest_forest - - def gather_forests( - self, - ) -> list[android_accessibility_forest_pb2.AndroidAccessibilityForest]: - forests = [] - with self._lock_forests: - forests = self._received_forests - self._received_forests = [] - return forests - - def gather_events(self) -> list[a11y_pb2.EventRequest]: - events = [] - with self._lock_events: - events = self._received_events - self._received_events = [] - return events - - def pause_and_clear(self) -> None: - """Temporarily stop receiving events/forests and clear the queue. - - Used when resetting the environment; in this case: - - all events/forests that have been received since last timestep are things - that happened in the last episode after its `LAST` timestep (so we should - ignore them, done by clearing the lists). - - we're about to receive a bunch of events/forests just as a result of - resetting the environment. We don't want to count these either; thus we - temporarily stop receiving new ones. - """ - self._paused = True - with self._lock_forests: - self._received_forests = [] - with self._lock_events: - self._received_events = [] - - def resume(self) -> None: - """Start receiving events/forests (e.g., after a reset).""" - self._paused = False - - def _process_event(self, event: a11y_pb2.EventRequest) -> None: - """Adds the given event to the internal buffer of events.""" - - if not self._paused: - with self._lock_events: - self._received_events.append(event) - - def _process_forest( - self, forest: android_accessibility_forest_pb2.AndroidAccessibilityForest - ) -> None: - """Adds the given forest to the internal buffer of forests.""" - - if not self._paused: - with self._lock_forests: - if self._latest_forest_only: - self._received_forests = [forest] - else: - self._received_forests.append(forest) diff --git a/android_env/wrappers/a11y/a11y_servicer_test.py b/android_env/wrappers/a11y/a11y_servicer_test.py deleted file mode 100644 index a61dd28e..00000000 --- a/android_env/wrappers/a11y/a11y_servicer_test.py +++ /dev/null @@ -1,224 +0,0 @@ -# coding=utf-8 -# Copyright 2026 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for a11y_servicer.""" - -import asyncio -from collections.abc import AsyncIterator, Iterable -from typing import TypeVar -from unittest import IsolatedAsyncioTestCase, mock - -from absl.testing import absltest -from absl.testing import parameterized -from android_env.proto.a11y import a11y_pb2 -from android_env.proto.a11y import android_accessibility_forest_pb2 -from android_env.wrappers.a11y import a11y_servicer -import grpc - - -_T = TypeVar('_T') - - -async def _aiter(xs: Iterable[_T]) -> AsyncIterator[_T]: - """Utility to make an AsyncIterator from Iterable.""" - - for x in xs: - yield x - - -def one_window_one_node_forest() -> ( - android_accessibility_forest_pb2.AndroidAccessibilityForest -): - forest = android_accessibility_forest_pb2.AndroidAccessibilityForest() - window = forest.windows.add() - node = window.tree.nodes.add() - node.class_name = 'foo' - node.is_clickable = True - node.hint_text = 'Foo hint' - return forest - - -def one_window_two_nodes_forest() -> ( - android_accessibility_forest_pb2.AndroidAccessibilityForest -): - forest = android_accessibility_forest_pb2.AndroidAccessibilityForest() - window = forest.windows.add() - node = window.tree.nodes.add() - node.class_name = 'bar' - node.is_clickable = True - node.hint_text = 'Bar hint' - node = window.tree.nodes.add() - node.class_name = 'bar' - node.is_clickable = False - node.hint_text = 'Bar hint 2' - return forest - - -def empty_dict() -> dict[str, str]: - return {} - - -def single_item_dict_with_special_chars() -> dict[str, str]: - return {'foo': 'bar\r\t\nbaz'} - - -class A11yServicerTest(parameterized.TestCase, IsolatedAsyncioTestCase): - - def test_servicer_sendforest(self): - mock_context = mock.create_autospec(grpc.ServicerContext, instance=True) - servicer = a11y_servicer.A11yServicer() - servicer.resume() - response = servicer.SendForest(one_window_one_node_forest(), mock_context) - self.assertEqual(response.error, '') - response = servicer.SendForest(one_window_two_nodes_forest(), mock_context) - self.assertEqual(response.error, '') - forests = servicer.gather_forests() - self.assertLen(forests, 2) - self.assertEqual(forests[0], one_window_one_node_forest()) - self.assertEqual(forests[1], one_window_two_nodes_forest()) - - async def test_servicer_bidi_forests(self): - """Checks that the bidirectional interface accepts forests.""" - - # Arrange. - mock_context = mock.create_autospec(grpc.ServicerContext, instance=True) - servicer = a11y_servicer.A11yServicer() - - # Act. - servicer.resume() - responses = [ - x - async for x in servicer.Bidi( - _aiter([ - a11y_pb2.ClientToServer( - event=a11y_pb2.EventRequest( - event=single_item_dict_with_special_chars() - ) - ), - a11y_pb2.ClientToServer(forest=one_window_two_nodes_forest()), - ]), - mock_context, - ) - ] - forest = await servicer.get_forest() - - # Assert. - self.assertEqual(responses[0], a11y_pb2.ServerToClient()) - self.assertEqual(responses[1], a11y_pb2.ServerToClient()) - self.assertIsNotNone(forest) - self.assertEqual(forest, one_window_two_nodes_forest()) - - def test_servicer_sendforest_latest_only(self): - mock_context = mock.create_autospec(grpc.ServicerContext, instance=True) - servicer = a11y_servicer.A11yServicer(latest_forest_only=True) - servicer.resume() - response = servicer.SendForest(one_window_one_node_forest(), mock_context) - self.assertEqual(response.error, '') - response = servicer.SendForest(one_window_two_nodes_forest(), mock_context) - self.assertEqual(response.error, '') - forests = servicer.gather_forests() - self.assertLen(forests, 1) - self.assertEqual(forests[0], one_window_two_nodes_forest()) - - def test_servicer_sendevent(self): - mock_context = mock.create_autospec(grpc.ServicerContext, instance=True) - servicer = a11y_servicer.A11yServicer() - servicer.resume() - response = servicer.SendEvent( - a11y_pb2.EventRequest(event=empty_dict()), mock_context - ) - self.assertEqual(response.error, '') - response = servicer.SendEvent( - a11y_pb2.EventRequest(event=single_item_dict_with_special_chars()), - mock_context, - ) - self.assertEqual(response.error, '') - events = servicer.gather_events() - self.assertLen(events, 2) - self.assertEqual(events[0].event, empty_dict()) - self.assertEqual(events[1].event, single_item_dict_with_special_chars()) - - async def test_servicer_bidi_events(self): - """Checks that the bidirectional interface accepts events.""" - - # Arrange. - mock_context = mock.create_autospec(grpc.ServicerContext, instance=True) - servicer = a11y_servicer.A11yServicer() - - # Act. - servicer.resume() - responses = [ - x - async for x in servicer.Bidi( - _aiter([ - a11y_pb2.ClientToServer( - event=a11y_pb2.EventRequest(event=empty_dict()) - ), - a11y_pb2.ClientToServer( - event=a11y_pb2.EventRequest( - event=single_item_dict_with_special_chars() - ) - ), - ]), - mock_context, - ) - ] - events = servicer.gather_events() - - # Assert. - self.assertEqual(responses[0], a11y_pb2.ServerToClient()) - self.assertEqual(responses[1], a11y_pb2.ServerToClient()) - self.assertLen(events, 2) - self.assertEqual(events[0].event, empty_dict()) - self.assertEqual(events[1].event, single_item_dict_with_special_chars()) - - def test_servicer_pause_and_clear_pauses(self): - mock_context = mock.create_autospec(grpc.ServicerContext, instance=True) - servicer = a11y_servicer.A11yServicer() - servicer.resume() - servicer.pause_and_clear() - response = servicer.SendEvent( - a11y_pb2.EventRequest(event=empty_dict()), mock_context - ) - self.assertEqual(response.error, '') - response = servicer.SendForest(one_window_one_node_forest(), mock_context) - self.assertEqual(response.error, '') - events = servicer.gather_events() - self.assertEmpty(events) - forests = servicer.gather_forests() - self.assertEmpty(forests) - - def test_servicer_pause_and_clear_clears(self): - mock_context = mock.create_autospec(grpc.ServicerContext, instance=True) - servicer = a11y_servicer.A11yServicer() - servicer.resume() - response = servicer.SendEvent( - a11y_pb2.EventRequest(event=empty_dict()), mock_context - ) - self.assertEqual(response.error, '') - response = servicer.SendForest(one_window_one_node_forest(), mock_context) - self.assertEqual( - response.error, - '', - ) - servicer.pause_and_clear() - events = servicer.gather_events() - self.assertEmpty(events) - forests = servicer.gather_forests() - self.assertEmpty(forests) - - -if __name__ == '__main__': - absltest.main() diff --git a/android_env/wrappers/a11y_grpc_wrapper.py b/android_env/wrappers/a11y_grpc_wrapper.py deleted file mode 100644 index b58aad70..00000000 --- a/android_env/wrappers/a11y_grpc_wrapper.py +++ /dev/null @@ -1,511 +0,0 @@ -# coding=utf-8 -# Copyright 2026 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Wraps AndroidEnv to retrieve accessibility messages from gRPC.""" - -from concurrent import futures -import time -from typing import Any - -import urllib.request - -from absl import logging -from android_env import env_interface -from android_env.components import action_type as android_action_type_lib -from android_env.proto import adb_pb2 -from android_env.proto.a11y import a11y_pb2_grpc -from android_env.wrappers import base_wrapper -from android_env.wrappers.a11y import a11y_events -from android_env.wrappers.a11y import a11y_forests -from android_env.wrappers.a11y import a11y_servicer -import dm_env -import grpc -import numpy as np -import portpicker - - -def _get_accessibility_forwarder_apk() -> bytes: - logging.info('Downloading accessibility forwarder apk....') - with urllib.request.urlopen( - 'https://storage.googleapis.com/android_env-tasks/2024.05.13-accessibility_forwarder.apk' - ) as response: - return response.read() - - -class EnableNetworkingError(ValueError): - pass - - -class A11yGrpcWrapper(base_wrapper.BaseWrapper): - """Wrapper which receives A11y events and forests over gRPC. - - A11y forest protobufs and event dicts are sent from the Android emulator via - gRPC from the `AccessibilityForwarder` (for use in developing reward - functions, etc). This wrapper constructs a server which receives these - messages and channels them into `task_extras`. - - The downside of forwarding this information through gRPC is that no messages - will be sent if networking is turned off (e.g., if the AVD is in airplane - mode). To mitigate this problem, the `AccessibilityForwarder` logs an error - message if it fails to contact the server. This wrapper monitors the logs for - such error messages, and attempts (in another thread, to not block environment - transitions) to reconnect the AVD to the network. If this fails to fix the - problem, this wrapper ends the episode. - - This wrapper is implemented to be robust to multiple upstream callers of - `task_extras`, and to ensure they each receive the same extras at every - timestep. Thus, the logic is the following: - * New a11y events/forests are fetched during `reset` and `step`, *not* during - `task_extras()` calls. - * If no one has called `task_extras()` since the last `step` or `reset`, the - extras are accumulated (so that no extras are missed because someone called - `step()` twice without calling `task_extras()`). - * If someone *has* called `task_extras()` since last step, the newly fetched - extras replace the old extras. - """ - - def __init__( - self, - env: env_interface.AndroidEnvInterface, - disable_other_network_traffic: bool = False, - install_a11y_forwarding: bool = False, - start_a11y_service: bool = True, - enable_a11y_tree_info: bool = False, - add_latest_a11y_info_to_obs: bool = False, - a11y_info_timeout: float | None = None, - max_enable_networking_attempts: int = 10, - latest_a11y_info_only: bool = False, - grpc_server_ip: str = '10.0.2.2', - port: int | None = None, - ): - """Initializes wrapper. - - Args: - env: Environment to wrap. - disable_other_network_traffic: When True, all network traffic, other than - the connection to the servicer, is disabled. NOTE: This requires root - access on the device (i.e. it uses the `su` command). An - `AdbControllerError` exception will be raised if the underlying command - fails. - install_a11y_forwarding: If True, the wrapper handles the installation of - all packages required for the servicer to collect a11y information. - start_a11y_service: If True, starts the a11y forwarding services. NOTE: - The packages must be installed beforehand, e.g., using the - install_a11y_forwarding flag. - enable_a11y_tree_info: When False, this wrapper collects only a11y events - and not a11y tree. - add_latest_a11y_info_to_obs: When True, the latest observed a11y forest is - added to the observation. - a11y_info_timeout: When larger than zero and add_latest_a11y_info_to_obs - is set to True, the wrapper will wait the corresponding amount of time, - measured in seconds, to collect the latest a11y forest. - max_enable_networking_attempts: When the a11y gRPC service fails to - provide a11y information, we attempt this many times to re-enable the - networking. If all these attempts fail, fetching task_extras will raise - an EnableNetworkingError. - latest_a11y_info_only: When True, the a11y servicer is setup to save only - the latest tree it has received from the Android app. - grpc_server_ip: The IP address of the gRPC server which will be - broadcasted to the AccessibilityForwarder app where it should log the - a11y info. By default, this is set to the IP address of the AVD's host - machine which is 10.0.2.2: See - https://developer.android.com/studio/run/emulator-networking#networkaddresses. - port: Port for the gRPC server. If not specified, a free port will be - allocated automatically. - """ - self._env = env - self._grpc_server_ip = grpc_server_ip - if install_a11y_forwarding: - self._install_a11y_forwarding_apk() - time.sleep(10.0) - if start_a11y_service: - self._start_a11y_services() - time.sleep(3.0) - if enable_a11y_tree_info: - self._enable_a11y_tree_logs() - self._relaunch_count = 0 - self._server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - self._servicer = a11y_servicer.A11yServicer( - latest_forest_only=latest_a11y_info_only - ) - a11y_pb2_grpc.add_A11yServiceServicer_to_server( - self._servicer, self._server - ) - server_credentials = grpc.local_server_credentials() - self._port = port or portpicker.pick_unused_port() - logging.info('Using port %s', self._port) - uri_address = f'[::]:{self._port}' - self._server.add_secure_port(uri_address, server_credentials) - logging.info('Starting server') - self._server.start() - logging.info('Server now running.') - - self._max_enable_networking_attempts = max_enable_networking_attempts - self._reset_enable_networking_attempts() - - self._disable_other_network_traffic = disable_other_network_traffic - self._should_accumulate = False - self._accumulated_extras = None - self._add_latest_a11y_info_to_obs = add_latest_a11y_info_to_obs - self._a11y_info_timeout = a11y_info_timeout - self._parent_action_spec = self._env.action_spec() - if self._a11y_info_timeout is not None and self._a11y_info_timeout > 0.0: - if 'action_type' not in self._parent_action_spec.keys(): - raise ValueError( - 'action_type not in the parent action spec: ' - f'{self._parent_action_spec}. This is a strong requirement when ' - f'a11y_info_timeout = {a11y_info_timeout} > 0' - ) - - def _start_a11y_services(self) -> None: - """Starts the accessibility forwarder services. - - Raises: - RuntimeError: If accessibility service is not started. - """ - start_service_request = adb_pb2.AdbRequest( - settings=adb_pb2.AdbRequest.SettingsRequest( - name_space=adb_pb2.AdbRequest.SettingsRequest.Namespace.SECURE, - put=adb_pb2.AdbRequest.SettingsRequest.Put( - key='enabled_accessibility_services', - value=( - 'com.google.androidenv.accessibilityforwarder/com.google.' - 'androidenv.accessibilityforwarder.AccessibilityForwarder' - ), - ), - ) - ) - start_service_response = self._env.execute_adb_call(start_service_request) - if start_service_response.status != adb_pb2.AdbResponse.Status.OK: - raise RuntimeError( - 'Could not start accessibility forwarder ' - 'service: ' - f'{start_service_response}.' - ) - - def _install_a11y_forwarding_apk(self) -> None: - """Enables accessibility information forwarding.""" - a11y_fwd_apk = _get_accessibility_forwarder_apk() - # Install and setup the Accesssibility Forwarder. - install_request = adb_pb2.AdbRequest( - install_apk=adb_pb2.AdbRequest.InstallApk( - blob=adb_pb2.AdbRequest.InstallApk.Blob(contents=a11y_fwd_apk), - ) - ) - install_response = self._env.execute_adb_call(install_request) - if install_response.status != adb_pb2.AdbResponse.Status.OK: - raise ValueError( - f'Could not install accessibility_forwarder.apk: {install_response}.' - ) - - def _enable_a11y_tree_logs(self) -> None: - enable_tree_logs_request = adb_pb2.AdbRequest( - send_broadcast=adb_pb2.AdbRequest.SendBroadcast( - action=( - 'accessibility_forwarder.intent.action.' - 'ENABLE_ACCESSIBILITY_TREE_LOGS' - ), - component=( - 'com.google.androidenv.accessibilityforwarder/com.google.androidenv.accessibilityforwarder.FlagsBroadcastReceiver' - ), - ) - ) - enable_tree_logs_response = self._env.execute_adb_call( - enable_tree_logs_request - ) - if enable_tree_logs_response.status != adb_pb2.AdbResponse.Status.OK: - raise ValueError( - 'Could not enable accessibility tree logging: ' - f'{enable_tree_logs_response}.' - ) - - def _reset_enable_networking_attempts(self) -> None: - self._enable_networking_attempts_left = self._max_enable_networking_attempts - self._enabling_networking_future = None - self._a11y_exception = None - - def get_port(self): - return self._port - - def close(self): - self._server.stop(None) - logging.info('gRPC server stopped') - self._env.close() - - def attempt_enable_networking(self) -> None: - """Attempts to turn on networking within the Android device. - - Attempt to turn on the networking in the Android device, by: - - turning off airplane mode; - - turning on the wifi connection. - """ - self.execute_adb_call( - adb_pb2.AdbRequest( - settings=adb_pb2.AdbRequest.SettingsRequest( - name_space=adb_pb2.AdbRequest.SettingsRequest.Namespace.GLOBAL, - put=adb_pb2.AdbRequest.SettingsRequest.Put( - key='airplane_mode_on', value='0' - ), - ) - ) - ) - time.sleep(1.0) - self.execute_adb_call( - adb_pb2.AdbRequest( - generic=adb_pb2.AdbRequest.GenericRequest( - args=[ - 'shell', - 'svc', - 'wifi', - 'enable', - ] - ) - ) - ) - time.sleep(1.0) - - def _configure_grpc(self) -> None: - """Configure networking and set the gRPC ip and port on AVD or device.""" - - if self._disable_other_network_traffic: - self.execute_adb_call( - adb_pb2.AdbRequest( - generic=adb_pb2.AdbRequest.GenericRequest( - args=[ - 'shell', - 'su', - '0', - 'iptables', - '-A', - 'OUTPUT', - '-p', - 'tcp', - '-d', - self._grpc_server_ip, - '--dport', - str(self._port), - '-j', - 'ACCEPT', - ] - ) - ) - ) - time.sleep(3.0) - self.execute_adb_call( - adb_pb2.AdbRequest( - generic=adb_pb2.AdbRequest.GenericRequest( - args=[ - 'shell', - 'su', - '0', - 'iptables', - '-A', - 'OUTPUT', - '-j', - 'DROP', - ] - ) - ) - ) - time.sleep(3.0) - - self.execute_adb_call( - adb_pb2.AdbRequest( - settings=adb_pb2.AdbRequest.SettingsRequest( - name_space=adb_pb2.AdbRequest.SettingsRequest.Namespace.GLOBAL, - put=adb_pb2.AdbRequest.SettingsRequest.Put( - key='no_proxy', value=f'{self._grpc_server_ip}:{self._port}' - ), - ) - ) - ) - self.attempt_enable_networking() - self.execute_adb_call( - adb_pb2.AdbRequest( - send_broadcast=adb_pb2.AdbRequest.SendBroadcast( - action=( - 'accessibility_forwarder.intent.action.SET_GRPC --ei' - f' "port" {self._port} --es "host" {self._grpc_server_ip}' - ), - component=( - 'com.google.androidenv.accessibilityforwarder/com.google.androidenv.accessibilityforwarder.FlagsBroadcastReceiver' - ), - ) - ) - ) - - def _accumulate_and_return_a11y_info( - self, timer: float | None = None, get_env_observation: bool = True - ) -> dict[str, Any]: - """Accumulates and returns the latest a11y tree info and observation. - - Args: - timer: If larger than 0, the system will wait this long for a11y info to - accumulate before it returns a value. - get_env_observation: If False, the corresponding observation is not - introduced here. - - Returns: - a dict with a11y forest under key 'a11y_forest'. All other fields will - provide the observation, if requested. - """ - timer = timer or 0.0 - if timer > 0.0: - time.sleep(timer) - - if get_env_observation: - # Fetch observation. - new_ts = self._env.step({ - 'action_type': np.array( - android_action_type_lib.ActionType.REPEAT, - dtype=self._parent_action_spec['action_type'].dtype, - ), - }) - observation = new_ts.observation - else: - observation = {} - - extras = self.accumulate_new_extras() - forests = a11y_forests.extract_forests_from_task_extras(extras) - if forests: - observation['a11y_forest'] = forests[-1] - else: - observation['a11y_forest'] = None - return observation - - def _fetch_task_extras_and_update_observation( - self, observation: dict[str, Any], timeout: float = 0.0 - ) -> dict[str, Any]: - if timeout > 0.0: - observation = self._accumulate_and_return_a11y_info( - timeout, get_env_observation=True - ) - if not self._add_latest_a11y_info_to_obs: - observation.pop('a11y_forest') - else: - new_obs = self._accumulate_and_return_a11y_info(get_env_observation=False) - if self._add_latest_a11y_info_to_obs: - observation.update(new_obs) - return observation - - def reset(self) -> dm_env.TimeStep: - self._reset_enable_networking_attempts() - self._servicer.pause_and_clear() - timestep = self._env.reset() - self._servicer.resume() - if self._env.stats()['relaunch_count'] > self._relaunch_count: - self._configure_grpc() - self._relaunch_count = self._env.stats()['relaunch_count'] - self._accumulated_extras = {} - timeout = self._a11y_info_timeout or 0.0 - new_observation = self._fetch_task_extras_and_update_observation( - timestep.observation, timeout - ) - timestep = timestep._replace(observation=new_observation) - return timestep - - def step(self, action: Any) -> dm_env.TimeStep: - timeout = float(action.pop('wait_time', self._a11y_info_timeout or 0.0)) - timestep = self._env.step(action) - new_observation = self._fetch_task_extras_and_update_observation( - timestep.observation, timeout=timeout - ) - timestep = timestep._replace(observation=new_observation) - return timestep - - def accumulate_new_extras(self) -> dict[str, Any]: - new_extras = self._fetch_task_extras() - if self._should_accumulate: - assert self._accumulated_extras is not None - for key in new_extras: - if key in self._accumulated_extras: - self._accumulated_extras[key] = np.concatenate( - (self._accumulated_extras[key], new_extras[key]), axis=0 - ) - else: - self._accumulated_extras[key] = new_extras[key] - else: - self._accumulated_extras = new_extras - self._should_accumulate = True - return self._accumulated_extras - - def _fetch_task_extras(self) -> dict[str, Any]: - """Fetches task_extras from the services. - - NOTE: If you want to access the latest a11y information, please use - accumulate_and_return_a11y_info instead. This function has the side effect - of clearing the content from the servicer, hence all the a11y info returned - here won't be accumulated. - - Returns: - A dict with the corresponding task_extras. - - Raises: - EnableNetworkingError: after a fixed number of attempts to revive the a11y - services by re-enabling the network connection. - """ - base_extras = self._env.task_extras(latest_only=False).copy() - # If the previous future is done, reset it to the initial state. - if ( - self._enabling_networking_future is not None - and self._enabling_networking_future.done() - ): - self._enabling_networking_future = None - self._enable_networking_attempts_left -= 1 - logging.info('Finished enabling networking.') - - if ( - self._enabling_networking_future is None - and 'exception' in base_extras - and base_extras['exception'].shape[0] - ): - self._a11y_exception = base_extras['exception'] - logging.warning( - 'AccessibilityForwarder logged exceptions: %s', self._a11y_exception - ) - if self._enable_networking_attempts_left > 0: - logging.warning( - 'Attempting to enable networking. %s attempts left.', - self._enable_networking_attempts_left - 1, - ) - executor = futures.ThreadPoolExecutor(max_workers=1) - self._enabling_networking_future = executor.submit( - self.attempt_enable_networking - ) - else: - raise EnableNetworkingError( - 'A11y service failed multiple times with' - f' exception.{self._a11y_exception}.' - ) - - forests = self._servicer.gather_forests() - if forests: - base_extras.update(a11y_forests.package_forests_to_task_extras(forests)) - self._reset_enable_networking_attempts() - events = self._servicer.gather_events() - if events: - base_extras.update(a11y_events.package_events_to_task_extras(events)) - self._reset_enable_networking_attempts() - return base_extras - - def task_extras(self, latest_only: bool = False) -> dict[str, Any]: - if self._accumulated_extras is None: - raise RuntimeError('You must call .reset() before calling .task_extras()') - self._should_accumulate = False - extras = self._accumulated_extras.copy() - if latest_only: - a11y_events.keep_latest_event_only(extras) - a11y_forests.keep_latest_forest_only(extras) - return extras diff --git a/android_env/wrappers/a11y_grpc_wrapper_test.py b/android_env/wrappers/a11y_grpc_wrapper_test.py deleted file mode 100644 index 9a171eb7..00000000 --- a/android_env/wrappers/a11y_grpc_wrapper_test.py +++ /dev/null @@ -1,967 +0,0 @@ -# coding=utf-8 -# Copyright 2026 DeepMind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for a11y_grpc_wrapper.""" - -import time -from unittest import mock - -from absl.testing import absltest -from absl.testing import parameterized -from android_env import env_interface -from android_env.proto import adb_pb2 -from android_env.proto.a11y import a11y_pb2 -from android_env.proto.a11y import a11y_pb2_grpc -from android_env.proto.a11y import android_accessibility_forest_pb2 -from android_env.wrappers import a11y_grpc_wrapper -import dm_env -import grpc -import numpy as np -import portpicker - - -def empty_forest() -> ( - android_accessibility_forest_pb2.AndroidAccessibilityForest -): - return android_accessibility_forest_pb2.AndroidAccessibilityForest() - - -def one_empty_window_forest() -> ( - android_accessibility_forest_pb2.AndroidAccessibilityForest -): - forest = android_accessibility_forest_pb2.AndroidAccessibilityForest() - _ = forest.windows.add() - return forest - - -def one_window_one_node_forest() -> ( - android_accessibility_forest_pb2.AndroidAccessibilityForest -): - forest = android_accessibility_forest_pb2.AndroidAccessibilityForest() - window = forest.windows.add() - node = window.tree.nodes.add() - node.class_name = 'foo' - node.is_clickable = True - node.hint_text = 'Foo hint' - return forest - - -def one_window_two_nodes_forest() -> ( - android_accessibility_forest_pb2.AndroidAccessibilityForest -): - forest = android_accessibility_forest_pb2.AndroidAccessibilityForest() - window = forest.windows.add() - node = window.tree.nodes.add() - node.class_name = 'bar' - node.is_clickable = True - node.hint_text = 'Bar hint' - node = window.tree.nodes.add() - node.class_name = 'bar' - node.is_clickable = False - node.hint_text = 'Bar hint 2' - return forest - - -def three_windows_forest() -> ( - android_accessibility_forest_pb2.AndroidAccessibilityForest -): - forest = android_accessibility_forest_pb2.AndroidAccessibilityForest() - _ = forest.windows.add() - window = forest.windows.add() - node = window.tree.nodes.add() - node.class_name = 'foo' - node.is_clickable = True - node.hint_text = 'hint' - window = forest.windows.add() - node = window.tree.nodes.add() - node.class_name = 'baz' - node.is_clickable = True - node.hint_text = 'hint' - node = window.tree.nodes.add() - node.class_name = 'foobar' - node.is_clickable = False - node.hint_text = 'hint' - return forest - - -def empty_dict() -> dict[str, str]: - return {} - - -def single_item_dict() -> dict[str, str]: - return {'foo': 'bar'} - - -def several_long_items_dict() -> dict[str, str]: - return { - 'first_key': 'Lorem ipsum ' * 100, - 'second_key': 'the beginning is the end is' * 100, - } - - -def single_item_dict_with_special_chars() -> dict[str, str]: - return {'foo': 'bar\r\t\nbaz'} - - -def _ok_response(): - return adb_pb2.AdbResponse(status=adb_pb2.AdbResponse.Status.OK) - - -class A11yGrpcWrapperTest(parameterized.TestCase): - - def test_server(self): - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.task_extras.return_value = {} - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper(base_env) - wrapped_env.reset() - channel_creds = grpc.local_channel_credentials() - with grpc.secure_channel( - f'[::]:{wrapped_env.get_port()}', channel_creds - ) as channel: - grpc.channel_ready_future(channel).result() - stub = a11y_pb2_grpc.A11yServiceStub(channel) - stub.SendForest(one_window_one_node_forest()) - stub.SendForest(one_window_two_nodes_forest()) - wrapped_env.step({}) - extras = wrapped_env.task_extras(latest_only=False) - self.assertIn('accessibility_tree', extras) - self.assertEqual(extras['accessibility_tree'].shape[0], 2) - - # tests of fetch_task_extras: - # exception occurs (ensure attempt to enable networking) and recovers - # exception occurs and enable networking doesn't help - # exception occurs twice but with a forest sent between - - @parameterized.named_parameters( - ('no_events_or_forests', [], []), - ( - 'no_events', - [], - [one_window_one_node_forest(), one_window_two_nodes_forest()], - ), - ('no_forests', [empty_dict(), single_item_dict()], []), - ( - 'events_and_forests', - [empty_dict(), single_item_dict()], - [one_window_one_node_forest(), one_window_two_nodes_forest()], - ), - ) - @mock.patch.object(time, 'sleep', autospec=True) - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_fetch_task_extras( - self, - received_events, - received_forests, - mock_server, - mock_add_servicer, - mock_sleep, - ): - del mock_server, mock_add_servicer, mock_sleep - mock_context = mock.create_autospec(grpc.ServicerContext, instance=True) - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.task_extras.return_value = { - 'foo': np.array(['bar', 'baz'], dtype='U'), - 'some_key': np.array(['some_value'], dtype='U'), - } - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper(base_env) - wrapped_env.reset() - for forest in received_forests: - wrapped_env._servicer.SendForest(forest, mock_context) - for event in received_events: - wrapped_env._servicer.SendEvent( - a11y_pb2.EventRequest(event=event), mock_context - ) - with mock.patch.object( - wrapped_env, 'attempt_enable_networking' - ) as mock_attempt_enable_networking: - extras = wrapped_env._fetch_task_extras() - mock_attempt_enable_networking.assert_not_called() - self.assertIn('foo', extras) - np.testing.assert_array_equal(extras['foo'], ['bar', 'baz']) - self.assertIn('some_key', extras) - np.testing.assert_array_equal(extras['some_key'], ['some_value']) - if received_events: - self.assertIn('full_event', extras) - self.assertLen(extras['full_event'], len(received_events)) - for i, event in enumerate(received_events): - event = a11y_pb2.EventRequest(event=event) - np.testing.assert_array_equal(extras['full_event'][i], event) - else: - self.assertNotIn('full_event', extras) - if received_forests: - self.assertIn('accessibility_tree', extras) - self.assertLen(extras['accessibility_tree'], len(received_forests)) - for i, forest in enumerate(received_forests): - np.testing.assert_array_equal(extras['accessibility_tree'][i], forest) - else: - self.assertNotIn('accessibility_tree', extras) - - @mock.patch.object(time, 'sleep', autospec=True) - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_fetch_task_extras_enable_networking( - self, - mock_server, - mock_add_servicer, - mock_sleep, - ): - del mock_server, mock_add_servicer, mock_sleep - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.task_extras.return_value = { - 'foo': np.array(['bar'], dtype='U'), - 'some_key': np.array(['some_value'], dtype='U'), - 'exception': np.array(['fake exception'], dtype='U'), - } - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper(base_env) - with mock.patch.object( - wrapped_env, 'attempt_enable_networking' - ) as mock_attempt_enable_networking: - extras = wrapped_env._fetch_task_extras() - self.assertNotIn('accessibility_tree', extras) - self.assertNotIn('full_event', extras) - future = wrapped_env._enabling_networking_future - if future is not None: - future.result() - mock_attempt_enable_networking.assert_called_once() - - @mock.patch.object(time, 'sleep', autospec=True) - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_fetch_task_extras_enable_networking_twice( - self, - mock_server, - mock_add_servicer, - mock_sleep, - ): - del mock_server, mock_add_servicer, mock_sleep - mock_context = mock.create_autospec(grpc.ServicerContext, instance=True) - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.task_extras.return_value = { - 'foo': np.array(['bar'], dtype='U'), - 'some_key': np.array(['some_value'], dtype='U'), - } - - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper(base_env) - wrapped_env.reset() - - base_env.task_extras.return_value = { - 'foo': np.array(['bar'], dtype='U'), - 'some_key': np.array(['some_value'], dtype='U'), - 'exception': np.array(['fake exception'], dtype='U'), - } - with mock.patch.object( - wrapped_env, 'attempt_enable_networking' - ) as mock_attempt_enable_networking: - extras = wrapped_env._fetch_task_extras() - self.assertNotIn('accessibility_tree', extras) - self.assertNotIn('full_event', extras) - future = wrapped_env._enabling_networking_future - if future is not None: - future.result() - mock_attempt_enable_networking.assert_called_once() - # Fixed networking; send a forest so the wrapper knows it worked. - wrapped_env._servicer.SendForest(one_window_one_node_forest(), mock_context) - base_env.task_extras.return_value = { - 'foo': np.array(['bar'], dtype='U'), - 'some_key': np.array(['some_value'], dtype='U'), - } - extras = wrapped_env._fetch_task_extras() - self.assertIn('accessibility_tree', extras) - self.assertEqual(extras['accessibility_tree'].shape[0], 1) - self.assertEqual( - extras['accessibility_tree'][0], one_window_one_node_forest() - ) - - base_env.task_extras.return_value = { - 'foo': np.array(['bar'], dtype='U'), - 'some_key': np.array(['some_value'], dtype='U'), - 'exception': np.array(['fake exception'], dtype='U'), - } - with mock.patch.object( - wrapped_env, 'attempt_enable_networking' - ) as mock_attempt_enable_networking: - extras = wrapped_env._fetch_task_extras() - self.assertNotIn('accessibility_tree', extras) - self.assertNotIn('full_event', extras) - future = wrapped_env._enabling_networking_future - if future is not None: - future.result() - mock_attempt_enable_networking.assert_called_once() - - @mock.patch.object(time, 'sleep', autospec=True) - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_task_extras_raises_a11y_info_exception( - self, mock_sleep, mock_add_servicer, mock_server - ): - del mock_server, mock_add_servicer, mock_sleep - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.task_extras.return_value = { - 'foo': np.array(['bar'], dtype='U'), - 'some_key': np.array(['some_value'], dtype='U'), - } - - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - base_env.reset.return_value = dm_env.restart(observation={'dummy': 42}) - base_env.step.return_value = dm_env.transition( - observation={'dummy': 42}, reward=0.0 - ) - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper( - base_env, - add_latest_a11y_info_to_obs=True, - max_enable_networking_attempts=1, - ) - wrapped_env.reset() - - base_env.task_extras.return_value = { - 'foo': np.array(['bar'], dtype='U'), - 'some_key': np.array(['some_value'], dtype='U'), - 'exception': np.array(['fake exception'], dtype='U'), - } - with mock.patch.object( - wrapped_env, 'attempt_enable_networking' - ) as mock_attempt_enable_networking: - extras = wrapped_env._fetch_task_extras() - self.assertNotIn('accessibility_tree', extras) - self.assertNotIn('full_event', extras) - # Wait for the the attempt to finish. - future = wrapped_env._enabling_networking_future - if future is not None: - future.result() - mock_attempt_enable_networking.assert_called_once() - # The _fetch_task_extras() call inside the next step will force a restart - self.assertRaises( - a11y_grpc_wrapper.EnableNetworkingError, wrapped_env.step, {} - ) - - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_configure_grpc( - self, - mock_server, - mock_add_servicer, - ): - del mock_server, mock_add_servicer - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.task_extras.return_value = { - 'foo': np.array(['bar'], dtype='U'), - 'some_key': np.array(['some_value'], dtype='U'), - } - - base_env.stats.return_value = {'relaunch_count': 1} - base_env.execute_adb_call.return_value = _ok_response() - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper(base_env) - with mock.patch.object( - wrapped_env, '_configure_grpc' - ) as mock_configure_grpc: - wrapped_env.reset() - mock_configure_grpc.assert_called_once() - - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(portpicker, 'pick_unused_port', return_value=1111) - @mock.patch.object(grpc, 'server', autospec=True) - def test_custom_port( - self, - mock_grpc_server, - mock_pick_unused_port, - unused_mock_add_servicer, - ): - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - - # Case 1: Pass a custom port. It should use it and NOT call portpicker. - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper(base_env, port=2222) - self.assertEqual(wrapped_env.get_port(), 2222) - mock_pick_unused_port.assert_not_called() - mock_grpc_server.return_value.add_secure_port.assert_called_with( - '[::]:2222', mock.ANY - ) - - # Reset mock for next case - mock_grpc_server.return_value.add_secure_port.reset_mock() - - # Case 2: Do not pass a port. It should call portpicker. - wrapped_env_default = a11y_grpc_wrapper.A11yGrpcWrapper(base_env) - self.assertEqual(wrapped_env_default.get_port(), 1111) - mock_pick_unused_port.assert_called_once() - mock_grpc_server.return_value.add_secure_port.assert_called_with( - '[::]:1111', mock.ANY - ) - - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_task_extras_raises_before_reset( - self, unused_mock_server, unused_mock_add_servicer - ): - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper(base_env) - with self.assertRaisesRegex( - RuntimeError, - r'You must call \.reset\(\) before calling \.task_extras\(\)', - ): - wrapped_env.task_extras(latest_only=False) - - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_extras_accumulate_between_steps( - self, mock_server, mock_add_servicer - ): - del mock_server, mock_add_servicer - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - base_env.reset.return_value = dm_env.restart(observation={'dummy': 42}) - base_env.step.return_value = dm_env.transition( - observation={'dummy': 42}, reward=0.0 - ) - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper( - base_env, add_latest_a11y_info_to_obs=True - ) - with mock.patch.object(wrapped_env, '_fetch_task_extras'): - wrapped_env._fetch_task_extras.return_value = { # pyrefly: ignore[missing-attribute] - 'full_event': np.array(single_item_dict(), ndmin=1, dtype=object), - 'accessibility_tree': np.array(empty_forest(), ndmin=1, dtype=object), - } - timestep = wrapped_env.reset() - self.assertIn('a11y_forest', timestep.observation) - self.assertEqual(timestep.observation['a11y_forest'], empty_forest()) - wrapped_env._fetch_task_extras.return_value = { # pyrefly: ignore[missing-attribute] - 'full_event': np.array(empty_dict(), ndmin=1, dtype=object), - 'accessibility_tree': np.array( - one_window_two_nodes_forest(), ndmin=1, dtype=object - ), - } - timestep = wrapped_env.step({}) - self.assertIn('a11y_forest', timestep.observation) - self.assertEqual( - timestep.observation['a11y_forest'], one_window_two_nodes_forest() - ) - timestep = wrapped_env.step({}) - self.assertIn('a11y_forest', timestep.observation) - self.assertEqual( - timestep.observation['a11y_forest'], one_window_two_nodes_forest() - ) - wrapped_env._fetch_task_extras.return_value = { # pyrefly: ignore[missing-attribute] - 'full_event': np.array(single_item_dict(), ndmin=1, dtype=object), - } - timestep = wrapped_env.step({}) - self.assertIn('a11y_forest', timestep.observation) - self.assertEqual( - timestep.observation['a11y_forest'], one_window_two_nodes_forest() - ) - expected_task_extras = { - 'full_event': np.array( - [ - single_item_dict(), - empty_dict(), - empty_dict(), - single_item_dict(), - ], - dtype=object, - ), - 'accessibility_tree': np.array( - [ - empty_forest(), - one_window_two_nodes_forest(), - one_window_two_nodes_forest(), - ], - dtype=object, - ), - } - expected_task_extras_latest = { - 'full_event': np.array([single_item_dict()], dtype=object), - 'accessibility_tree': np.array( - [one_window_two_nodes_forest()], dtype=object - ), - } - task_extras = wrapped_env.task_extras(latest_only=False) - np.testing.assert_equal( - task_extras['full_event'], expected_task_extras['full_event'] - ) - np.testing.assert_equal( - task_extras['accessibility_tree'], - expected_task_extras['accessibility_tree'], - ) - - task_extras = wrapped_env.task_extras(latest_only=True) - np.testing.assert_equal( - task_extras['full_event'], expected_task_extras_latest['full_event'] - ) - np.testing.assert_equal( - task_extras['accessibility_tree'], - expected_task_extras_latest['accessibility_tree'], - ) - - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_a11y_info_disabled( - self, - unused_mock_server, - unused_mock_add_servicer, - ): - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.action_spec.return_value = { - 'action_type': dm_env.specs.Array(shape=(), dtype=np.int32) - } - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - base_env.reset.return_value = dm_env.restart(observation={'dummy': 42}) - base_env.step.return_value = dm_env.transition( - observation={'dummy': 42}, reward=0.0 - ) - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper( - base_env, add_latest_a11y_info_to_obs=False, a11y_info_timeout=1.0 - ) - with mock.patch.object(wrapped_env, '_fetch_task_extras'): - wrapped_env._fetch_task_extras.return_value = { # pyrefly: ignore[missing-attribute] - 'accessibility_tree': np.array(empty_forest(), ndmin=1, dtype=object), - } - timestep = wrapped_env.reset() - self.assertNotIn('a11y_forest', timestep.observation) - timestep = wrapped_env.step({}) - self.assertNotIn('a11y_forest', timestep.observation) - - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_a11y_info_with_timer_info_present( - self, - unused_mock_server, - unused_mock_add_servicer, - ): - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.action_spec.return_value = { - 'action_type': dm_env.specs.Array(shape=(), dtype=np.int32) - } - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - base_env.reset.return_value = dm_env.restart(observation={'dummy': 42}) - base_env.step.return_value = dm_env.transition( - observation={'dummy': 42}, reward=0.0 - ) - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper( - base_env, add_latest_a11y_info_to_obs=True, a11y_info_timeout=1.0 - ) - with mock.patch.object(wrapped_env, '_fetch_task_extras'): - wrapped_env._fetch_task_extras.side_effect = [{ # pyrefly: ignore[missing-attribute] - 'accessibility_tree': np.array(empty_forest(), ndmin=1, dtype=object), - }] - timestep = wrapped_env.reset() - self.assertIn('a11y_forest', timestep.observation) - self.assertEqual(timestep.observation['a11y_forest'], empty_forest()) - - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - @mock.patch.object(time, 'sleep', autospec=True) - def test_a11y_info_with_timer_task_extra_returned( - self, unused_mock_server, unused_mock_add_servicer, unused_mock_sleep - ): - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.action_spec.return_value = { - 'action_type': dm_env.specs.Array(shape=(), dtype=np.int32) - } - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - base_env.reset.return_value = dm_env.restart(observation={'dummy': 42}) - base_env.step.return_value = dm_env.transition( - observation={'dummy': 42}, reward=0.0 - ) - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper( - base_env, add_latest_a11y_info_to_obs=True, a11y_info_timeout=1.0 - ) - with mock.patch.object(wrapped_env, '_fetch_task_extras'): - wrapped_env._fetch_task_extras.side_effect = [ # pyrefly: ignore[missing-attribute] - { - 'accessibility_tree': np.array( - empty_forest(), ndmin=1, dtype=object - ), - }, - ] - timestep = wrapped_env.reset() - self.assertIn('a11y_forest', timestep.observation) - self.assertEqual(timestep.observation['a11y_forest'], empty_forest()) - - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - @mock.patch.object(time, 'sleep', autospec=True) - def test_a11y_info_with_timer_from_action( - self, unused_mock_server, unused_mock_add_servicer, mock_sleep - ): - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.action_spec.return_value = { - 'action_type': dm_env.specs.Array(shape=(), dtype=np.int32) - } - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - base_env.reset.return_value = dm_env.restart(observation={'dummy': 42}) - base_env.step.return_value = dm_env.transition( - observation={'dummy': 42}, reward=0.0 - ) - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper( - base_env, add_latest_a11y_info_to_obs=True, a11y_info_timeout=0.0 - ) - with mock.patch.object(wrapped_env, '_fetch_task_extras'): - wrapped_env._fetch_task_extras.side_effect = [ # pyrefly: ignore[missing-attribute] - { - 'accessibility_tree': np.array( - empty_forest(), ndmin=1, dtype=object - ), - }, - ] - timestep = wrapped_env.step(action={'wait_time': 1.0}) - self.assertIn('a11y_forest', timestep.observation) - mock_sleep.assert_called_once() - self.assertEqual(timestep.observation['a11y_forest'], empty_forest()) - - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_task_extras_same_between_calls(self, mock_server, mock_add_servicer): - del mock_server, mock_add_servicer - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper(base_env) - expected_task_extras = { - 'full_event': np.array(single_item_dict(), ndmin=1, dtype=object), - 'accessibility_tree': np.array(empty_forest(), ndmin=1, dtype=object), - } - with mock.patch.object(wrapped_env, '_fetch_task_extras'): - wrapped_env._fetch_task_extras.return_value = expected_task_extras # pyrefly: ignore[missing-attribute] - wrapped_env.reset() - task_extras = wrapped_env.task_extras(latest_only=False) - np.testing.assert_equal( - task_extras['full_event'], expected_task_extras['full_event'] - ) - np.testing.assert_equal( - task_extras['accessibility_tree'], - expected_task_extras['accessibility_tree'], - ) - - task_extras = wrapped_env.task_extras(latest_only=False) - np.testing.assert_equal( - task_extras['full_event'], expected_task_extras['full_event'] - ) - np.testing.assert_equal( - task_extras['accessibility_tree'], - expected_task_extras['accessibility_tree'], - ) - - expected_task_extras = { - 'full_event': np.array(empty_dict(), ndmin=1, dtype=object), - 'accessibility_tree': np.array( - one_window_two_nodes_forest(), ndmin=1, dtype=object - ), - } - with mock.patch.object(wrapped_env, '_fetch_task_extras'): - wrapped_env._fetch_task_extras.return_value = expected_task_extras # pyrefly: ignore[missing-attribute] - wrapped_env.step({}) - task_extras = wrapped_env.task_extras(latest_only=False) - np.testing.assert_equal( - task_extras['full_event'], expected_task_extras['full_event'] - ) - np.testing.assert_equal( - task_extras['accessibility_tree'], - expected_task_extras['accessibility_tree'], - ) - - task_extras = wrapped_env.task_extras(latest_only=False) - np.testing.assert_equal( - task_extras['full_event'], expected_task_extras['full_event'] - ) - np.testing.assert_equal( - task_extras['accessibility_tree'], - expected_task_extras['accessibility_tree'], - ) - - @mock.patch.object( - a11y_pb2_grpc, 'add_A11yServiceServicer_to_server', autospec=True - ) - @mock.patch.object(grpc, 'server', autospec=True) - def test_task_extras_clear_if_called_between_step( - self, mock_server, mock_add_servicer - ): - del mock_server, mock_add_servicer - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.stats.return_value = {'relaunch_count': 0} - base_env.execute_adb_call.return_value = _ok_response() - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper(base_env) - with mock.patch.object(wrapped_env, '_fetch_task_extras'): - expected_task_extras = { - 'full_event': np.array(empty_dict(), ndmin=1, dtype=object), - 'accessibility_tree': np.array(empty_forest(), ndmin=1, dtype=object), - } - wrapped_env._fetch_task_extras.return_value = expected_task_extras # pyrefly: ignore[missing-attribute] - wrapped_env.reset() - task_extras = wrapped_env.task_extras(latest_only=False) - np.testing.assert_equal( - task_extras['full_event'], expected_task_extras['full_event'] - ) - np.testing.assert_equal( - task_extras['accessibility_tree'], - expected_task_extras['accessibility_tree'], - ) - - expected_task_extras = { - 'full_event': np.array(single_item_dict(), ndmin=1, dtype=object), - 'accessibility_tree': np.array(empty_forest(), ndmin=1, dtype=object), - } - wrapped_env._fetch_task_extras.return_value = expected_task_extras # pyrefly: ignore[missing-attribute] - wrapped_env.step({}) - task_extras = wrapped_env.task_extras(latest_only=False) - np.testing.assert_equal( - task_extras['full_event'], expected_task_extras['full_event'] - ) - np.testing.assert_equal( - task_extras['accessibility_tree'], - expected_task_extras['accessibility_tree'], - ) - expected_task_extras = { - 'full_event': np.array(empty_dict(), ndmin=1, dtype=object), - 'accessibility_tree': np.array( - one_window_two_nodes_forest(), ndmin=1, dtype=object - ), - } - wrapped_env._fetch_task_extras.return_value = expected_task_extras # pyrefly: ignore[missing-attribute] - wrapped_env.step({}) - task_extras = wrapped_env.task_extras(latest_only=False) - np.testing.assert_equal( - task_extras['full_event'], expected_task_extras['full_event'] - ) - np.testing.assert_equal( - task_extras['accessibility_tree'], - expected_task_extras['accessibility_tree'], - ) - - @parameterized.named_parameters( - ('none_true', False, False, False, 0), - ('only_install', True, False, False, 1), - ('only_start', False, True, False, 1), - ('only_enable_a11y_tree', False, False, True, 1), - ('install_and_start_no_a11y_tree', True, True, False, 2), - ('install_and_a11y_tree', True, False, True, 2), - ('start_and_a11y_tree', False, True, True, 2), - ('all_true', True, True, True, 3), - ) - @mock.patch.object(time, 'sleep', autospec=True) - def test_apk_install_and_start( - self, - install_a11y_forwarding: bool, - start_a11y_service: bool, - enable_a11y_tree_logs: bool, - expected_adb_calls: int, - unused_mock_sleep, - ): - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - - side_effects = [] - if install_a11y_forwarding: - side_effects.append(_ok_response()) # install response - if start_a11y_service: - side_effects.append(_ok_response()) # start service response - if enable_a11y_tree_logs: - side_effects.append(_ok_response()) # enable_tree_request - - base_env.execute_adb_call.side_effect = side_effects - - _ = a11y_grpc_wrapper.A11yGrpcWrapper( - base_env, - install_a11y_forwarding=install_a11y_forwarding, - start_a11y_service=start_a11y_service, - enable_a11y_tree_info=enable_a11y_tree_logs, - ) - self.assertEqual(base_env.execute_adb_call.call_count, expected_adb_calls) - - @mock.patch.object(time, 'sleep', autospec=True) - def test_component_and_start(self, unused_mock_sleep): - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - - side_effects = [] - side_effects.append(_ok_response()) # install response - side_effects.append(_ok_response()) # start service response - side_effects.append(_ok_response()) # enable_tree_request - - base_env.execute_adb_call.side_effect = side_effects - - _ = a11y_grpc_wrapper.A11yGrpcWrapper( - base_env, - install_a11y_forwarding=True, - start_a11y_service=True, - enable_a11y_tree_info=True, - ) - - # call_args returns a tuple of which the first member is a tuple containing - # the most recent args the mock was called with, and execute_adb_call only - # has one arg (so [0][0] to access the AdbRequest). - self.assertEqual( - base_env.execute_adb_call.call_args[0][0].send_broadcast.component, - 'com.google.androidenv.accessibilityforwarder/com.google.androidenv.accessibilityforwarder.FlagsBroadcastReceiver', - ) - - def test_broadcast_sent_default_grpc_server_ip(self): - """Tests that the broadcast sets the default grpc server ip.""" - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.execute_adb_call.return_value = _ok_response() - base_env.stats.return_value = {'relaunch_count': 1} - - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper( - env=base_env, - disable_other_network_traffic=False, - install_a11y_forwarding=False, - start_a11y_service=False, - enable_a11y_tree_info=False, - ) - wrapped_env.reset() - - self.assertStartsWith( - base_env.execute_adb_call.call_args[0][0].send_broadcast.action, - 'accessibility_forwarder.intent.action.SET_GRPC', - ) - - self.assertIn( - '--es "host" 10.0.2.2', - base_env.execute_adb_call.call_args[0][0].send_broadcast.action, - ) - - @parameterized.parameters(('127.0.0.1',), ('1.2.3.4',), 'localhost') - def test_broadcast_sent_custom_grpc_server_ip(self, grpc_server_ip): - """Tests that the broadcast sets the custom grpc server ip.""" - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.execute_adb_call.return_value = _ok_response() - base_env.stats.return_value = {'relaunch_count': 1} - - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper( - env=base_env, - disable_other_network_traffic=False, - install_a11y_forwarding=False, - start_a11y_service=False, - enable_a11y_tree_info=False, - grpc_server_ip=grpc_server_ip, - ) - wrapped_env.reset() - - self.assertStartsWith( - base_env.execute_adb_call.call_args[0][0].send_broadcast.action, - 'accessibility_forwarder.intent.action.SET_GRPC', - ) - self.assertIn( - f'--es "host" {grpc_server_ip}', - base_env.execute_adb_call.call_args[0][0].send_broadcast.action, - ) - - def test_broadcast_sent_port(self): - """Tests that the broadcast sets the correct port.""" - base_env = mock.create_autospec( - env_interface.AndroidEnvInterface, instance=True - ) - base_env.execute_adb_call.return_value = _ok_response() - base_env.stats.return_value = {'relaunch_count': 1} - - wrapped_env = a11y_grpc_wrapper.A11yGrpcWrapper( - env=base_env, - disable_other_network_traffic=False, - install_a11y_forwarding=False, - start_a11y_service=False, - enable_a11y_tree_info=False, - ) - wrapped_env.reset() - - self.assertStartsWith( - base_env.execute_adb_call.call_args[0][0].send_broadcast.action, - 'accessibility_forwarder.intent.action.SET_GRPC', - ) - self.assertIn( - f'--ei "port" {wrapped_env.get_port()}', - base_env.execute_adb_call.call_args[0][0].send_broadcast.action, - ) - - -if __name__ == '__main__': - absltest.main() diff --git a/android_env/wrappers/base_wrapper.py b/android_env/wrappers/base_wrapper.py index 8868e1ca..6fcb7f45 100644 --- a/android_env/wrappers/base_wrapper.py +++ b/android_env/wrappers/base_wrapper.py @@ -20,7 +20,6 @@ from absl import logging from android_env import env_interface from android_env.proto import adb_pb2 -from android_env.proto import state_pb2 import dm_env from dm_env import specs import numpy as np @@ -75,27 +74,6 @@ def stats(self) -> dict[str, Any]: info.update(self._wrapper_stats()) return info - def load_state( - self, request: state_pb2.LoadStateRequest - ) -> state_pb2.LoadStateResponse: - """Loads a state.""" - return self._env.load_state(request) - - def save_state( - self, request: state_pb2.SaveStateRequest - ) -> state_pb2.SaveStateResponse: - """Saves a state. - - Args: - request: A `SaveStateRequest` containing any parameters necessary to - specify how/what state to save. - - Returns: - A `SaveStateResponse` containing the status, error message (if - applicable), and any other relevant information. - """ - return self._env.save_state(request) - def execute_adb_call( self, adb_call: adb_pb2.AdbRequest ) -> adb_pb2.AdbResponse: diff --git a/android_env/wrappers/base_wrapper_test.py b/android_env/wrappers/base_wrapper_test.py index 9a8414a2..9c24adbd 100644 --- a/android_env/wrappers/base_wrapper_test.py +++ b/android_env/wrappers/base_wrapper_test.py @@ -20,7 +20,6 @@ from absl import logging from absl.testing import absltest from android_env import env_interface -from android_env.proto import state_pb2 from android_env.wrappers import base_wrapper @@ -67,22 +66,6 @@ def test_base_function_forwarding(self, mock_info): return_value=fake_raw_observation) self.assertEqual(fake_raw_observation, wrapped_env.raw_observation) - load_request = state_pb2.LoadStateRequest(args={}) - expected_response = state_pb2.LoadStateResponse( - status=state_pb2.LoadStateResponse.Status.OK - ) - base_env.load_state.return_value = expected_response - self.assertEqual(wrapped_env.load_state(load_request), expected_response) - base_env.load_state.assert_called_once_with(load_request) - - save_request = state_pb2.SaveStateRequest(args={}) - expected_response = state_pb2.SaveStateResponse( - status=state_pb2.SaveStateResponse.Status.OK - ) - base_env.save_state.return_value = expected_response - self.assertEqual(wrapped_env.save_state(save_request), expected_response) - base_env.save_state.assert_called_once_with(save_request) - wrapped_env.close() base_env.close.assert_called_once() diff --git a/py_grpc_library.bzl b/py_grpc_library.bzl deleted file mode 100644 index d97b86d4..00000000 --- a/py_grpc_library.bzl +++ /dev/null @@ -1,98 +0,0 @@ -"""Pure Python Proto and gRPC generation macros using prebuilt wheel binaries. - -RATIONALE & PERFORMANCE CONSIDERATIONS: ----------------------------------------- -These macros invoke `protoc_runner` (which wraps `grpc_tools.protoc` from -`@pip//grpcio_tools`) via a lightweight Bazel `genrule` instead of relying on -`rules_proto_grpc_python`. - -Why this is done: -1. `rules_proto_grpc_python` pulls in `@grpc~` C++ source dependencies (`cygrpc.so`, - `abseil-cpp`, and the `protobuf` C++ runtime), forcing Bazel to compile C++ source - code on cold checkouts. -2. Compiling `@grpc~` C++ targets takes ~17 minutes on cold checkouts. -3. By using the hermetic `//:protoc_runner` tool (`@pip//grpcio_tools`), proto and gRPC - code generation completes in under 1 second without compiling any C++ code, reducing - total cold checkout build times from ~17 minutes to ~1 minute. -""" - -load("@rules_python//python:defs.bzl", "py_library") - -def _dedup(seq): - seen = {} - return [x for x in seq if not (x in seen or seen.update({x: 1}))] - -def py_proto_library(name, srcs, proto_srcs = [], deps = [], visibility = ["//visibility:public"]): - """Generates _pb2.py message files for srcs using hermetic protoc_runner. - - Avoids compiling C++ protobuf source dependencies to keep cold build times fast. - - Args: - name: Target name for the generated py_library. - srcs: List of primary .proto files to generate _pb2.py for. - proto_srcs: Additional .proto files needed in the Bazel sandbox for imports. - deps: Additional Python library dependencies. - visibility: Visibility specifier for the generated py_library. - """ - outs = [s[:-6] + "_pb2.py" if s.endswith(".proto") else s + "_pb2.py" for s in srcs] - pkg_parts = native.package_name().split("/") - depth = "../" * len(pkg_parts) - root_import = "/".join([".."] * len(pkg_parts)) - gen_name = name + "_gen" - - all_srcs = _dedup(srcs + proto_srcs) - - native.genrule( - name = gen_name, - srcs = all_srcs, - outs = outs, - cmd = "$(location //:protoc_runner) -I. --python_out=$(RULEDIR)/" + depth + " " + " ".join(["$(location " + s + ")" for s in srcs]), - tools = ["//:protoc_runner"], - ) - - py_library( - name = name, - srcs = [":" + gen_name], - imports = [root_import, "."], - deps = ["@pip//protobuf"] + deps, - visibility = visibility, - ) - -def py_grpc_library(name, srcs, proto_srcs = [], deps = [], visibility = ["//visibility:public"]): - """Generates _pb2_grpc.py stub/servicer files for srcs using hermetic protoc_runner. - - Avoids compiling C++ gRPC source dependencies to keep cold build times fast. - - Args: - name: Target name for the generated py_library. - srcs: List of primary .proto files to generate _pb2_grpc.py for. - proto_srcs: Additional .proto files needed in the Bazel sandbox for imports. - deps: Additional Python library dependencies (typically including the matching py_proto_library). - visibility: Visibility specifier for the generated py_library. - """ - outs = [s[:-6] + "_pb2_grpc.py" if s.endswith(".proto") else s + "_pb2_grpc.py" for s in srcs] - pkg_parts = native.package_name().split("/") - depth = "../" * len(pkg_parts) - root_import = "/".join([".."] * len(pkg_parts)) - gen_name = name + "_gen" - - all_srcs = _dedup(srcs + proto_srcs) - - native.genrule( - name = gen_name, - srcs = all_srcs, - outs = outs, - cmd = "$(location //:protoc_runner) -I. --grpc_python_out=$(RULEDIR)/" + depth + " " + " ".join(["$(location " + s + ")" for s in srcs]), - tools = ["//:protoc_runner"], - ) - - py_library( - name = name, - srcs = [":" + gen_name], - imports = [root_import, "."], - deps = [ - "@pip//grpcio", - "@pip//protobuf", - ] + deps, - visibility = visibility, - ) diff --git a/py_proto_library.bzl b/py_proto_library.bzl new file mode 100644 index 00000000..b5630621 --- /dev/null +++ b/py_proto_library.bzl @@ -0,0 +1,53 @@ +"""Pure Python Proto generation macro using prebuilt wheel binaries. + +RATIONALE & PERFORMANCE CONSIDERATIONS: +---------------------------------------- +This macro invokes `protoc_runner` (which wraps `grpc_tools.protoc` from +`@pip//grpcio_tools`) via a lightweight Bazel `genrule` instead of compiling +protobuf `protoc` C++ source dependencies on cold checkouts. + +By using the hermetic `//:protoc_runner` tool (`@pip//grpcio_tools`), proto +code generation completes in under 1 second without compiling any C++ code. +""" + +load("@rules_python//python:defs.bzl", "py_library") + +def _dedup(seq): + seen = {} + return [x for x in seq if not (x in seen or seen.update({x: 1}))] + +def py_proto_library(name, srcs, proto_srcs = [], deps = [], visibility = ["//visibility:public"]): + """Generates _pb2.py message files for srcs using hermetic protoc_runner. + + Avoids compiling C++ protobuf source dependencies to keep cold build times fast. + + Args: + name: Target name for the generated py_library. + srcs: List of primary .proto files to generate _pb2.py for. + proto_srcs: Additional .proto files needed in the Bazel sandbox for imports. + deps: Additional Python library dependencies. + visibility: Visibility specifier for the generated py_library. + """ + outs = [s[:-6] + "_pb2.py" if s.endswith(".proto") else s + "_pb2.py" for s in srcs] + pkg_parts = native.package_name().split("/") + depth = "../" * len(pkg_parts) + root_import = "/".join([".."] * len(pkg_parts)) + gen_name = name + "_gen" + + all_srcs = _dedup(srcs + proto_srcs) + + native.genrule( + name = gen_name, + srcs = all_srcs, + outs = outs, + cmd = "$(location //:protoc_runner) -I. --python_out=$(RULEDIR)/" + depth + " " + " ".join(["$(location " + s + ")" for s in srcs]), + tools = ["//:protoc_runner"], + ) + + py_library( + name = name, + srcs = [":" + gen_name], + imports = [root_import, "."], + deps = ["@pip//protobuf"] + deps, + visibility = visibility, + ) diff --git a/pyproject.toml b/pyproject.toml index abb4a14e..720adeb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,17 +7,16 @@ build-backend = "setuptools.build_meta" [project] name = "android-env" -version = "1.3.0" +version = "2.0.0" description = "AndroidEnv environment and library for training agents." authors = [{name = "DeepMind"}] license = {file = "LICENSE"} -readme = {text = "Read the README at https://github.com/deepmind/android_env for more information.", content-type = "text/plain"} +readme = {text = "Read the README at https://github.com/google-deepmind/android_env for more information.", content-type = "text/plain"} keywords = ["Android", "OS", "reinforcement-learning"] requires-python = ">=3.10" dependencies = [ "absl-py>=2.1.0", "dm_env>=1.6", - "grpcio>=1.70.0", "numpy>=2.1.0", "portpicker>=1.6.0", "protobuf>=6,<8", @@ -32,7 +31,7 @@ testing = [ "pytype", ] acme = ["dm-acme"] -gym = ["gymnasium"] +gym = ["gymnasium>=0.29.1"] mcp = ["mcp>=1.26,<2"] playground = ["ml-collections"] @@ -40,6 +39,6 @@ playground = ["ml-collections"] "android_env.components" = ["*.textproto"] [project.urls] -repository = "https://github.com/deepmind/android_env" +repository = "https://github.com/google-deepmind/android_env" deepmind = "https://www.deepmind.com/publications/androidenv-the-android-learning-environment" arxiv = "https://arxiv.org/abs/2105.13231" diff --git a/requirements.txt b/requirements.txt index 73b18932..7a91b292 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ portpicker>=1.6.0 pillow>=10.4.0 gymnasium>=0.29.1 protobuf>=6,<8 -grpcio>=1.70.0 +# Build-time proto compiler for //:protoc_runner (not a runtime dependency). grpcio-tools>=1.70.0 farama-notifications>=0.0.4 typing-extensions>=4.11.0 @@ -25,6 +25,8 @@ cffi==2.1.1 click==8.5.0 cloudpickle==3.1.2 cryptography==50.0.2 +# Transitive dependency of grpcio-tools for Bazel pip.parse metadata wiring. +grpcio>=1.70.0 h11==0.16.0 httpcore==1.0.9 httpx==0.28.1