Back to build
Raw log | Switch to full mode | Login

/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py --blacklist-file /b/swarming/w/ir/cache/builder/src/out/bad_devices.json --known-devices-file /home/chrome-bot/.android/known_devices.json --adb-path /b/swarming/w/ir/cache/builder/src/third_party/android_tools/sdk/platform-tools/adb -v in dir /b/swarming/w/ir/cache/builder: allow_subannotations: False base_name: 'device_recovery' cmd: ['/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py', '--blacklist-file', '/b/swarming/w/ir/cache/builder/src/out/bad_devices.json', '--known-devices-file', '/home/chrome-bot/.android/known_devices.json', '--adb-path', '/b/swarming/w/ir/cache/builder/src/third_party/android_tools/sdk/platform-tools/adb', '-v'] cwd: '/b/swarming/w/ir/cache/builder' env: {'PATH': '/b/swarming/w/ir/cache/builder/src/third_party/android_tools/sdk/platform-tools:/b/swarming/w/ir/cache/builder/src/build/android:%(PATH)s'} env_prefixes: {} env_suffixes: {} infra_step: True name: 'device_recovery' nest_level: 0 ok_ret: frozenset([0]) trigger_specs: () full environment: ADB_LIBUSB: 0 BOTO_CONFIG: /b/swarming/w/ir/tmp/gsutil_task/.boto BUILDBUCKET_EXPERIMENTAL: FALSE CHROMIUM_REV: e41aeaed051c8cc401e0bce2a6d57dcc330808e6 CIPD_CACHE_DIR: /b/swarming/cipd_cache/cache CIPD_PROTOCOL: v2 DEBIAN_FRONTEND: noninteractive DEVSHELL_CLIENT_PORT: 59146 DOCKER_CONFIG: /b/swarming/w/ir/tmp/docker_cfg_task DOCKER_HOST_HOSTNAME: build6-b9.labs.chromium.org DOCKER_TMPDIR: /b/swarming/w/ir/tmp/docker_tmp_task GIT_CONFIG_NOSYSTEM: 1 GIT_TERMINAL_PROMPT: 0 HOME: /home/chrome-bot HOSTNAME: build6-b9--device2 INFRA_GIT_WRAPPER_HOME: /b/swarming/w/ir/tmp/git_home_task ISOLATED_CACHE_SIZE: 54287331328 LOGDOG_COORDINATOR_HOST: logs.chromium.org LOGDOG_STREAM_PREFIX: buildbucket/cr-buildbucket.appspot.com/8931036902136175840 LOGDOG_STREAM_PROJECT: libyuv LOGDOG_STREAM_SERVER_PATH: unix:/b/swarming/w/ir/tmp/ld.sock LOGNAME: chrome-bot LUCI_CONTEXT: /b/swarming/w/itLN43fB/luci_context.529989210 MAC_CHROMIUM_TMPDIR: /b/swarming/w/ir/tmp/t MAIL: /var/mail/chrome-bot NO_GCE_CHECK: False NUM_CONFIGURED_CONTAINERS: 7 PATH: /b/swarming/w/ir/cache/builder/src/third_party/android_tools/sdk/platform-tools:/b/swarming/w/ir/cache/builder/src/build/android:/b/swarming/w/ir/cipd_bin_packages:/b/swarming/w/ir/cipd_bin_packages/bin:/b/swarming/cipd_cache/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games PWD: /b/swarming/w/ir PYTHONIOENCODING: UTF-8 PYTHONUNBUFFERED: 1 SHELL: /bin/bash SHLVL: 3 SWARMING_BOT_ID: build6-b9--device2 SWARMING_HEADLESS: 1 SWARMING_SERVER: https://chromium-swarm.appspot.com SWARMING_TASK_ID: 40e9541e0eec4111 SWARM_URL: https://chromium-swarm.appspot.com/bot_code TEMP: /b/swarming/w/ir/tmp/t TEMPDIR: /b/swarming/w/ir/tmp/t TMP: /b/swarming/w/ir/tmp/t TMPDIR: /b/swarming/w/ir/tmp/t USER: chrome-bot VPYTHON_VIRTUALENV_ROOT: /b/swarming/w/ir/cache/vpython _: /b/swarming/w/ir/cipd_bin_packages/vpython I 0.008s Main Expected devices: I 0.008s Main 01ff3a86c26b4d26 I 0.008s Main 00c56ac971923a54 I 0.008s Main 00b1e3f189a332e5 I 0.008s Main 01e1be00ca881247 I 1.394s TimeoutThread-1-for-MainThread condition '<lambda>' met (0.0s) I 3.109s Main Resetting blacklist /b/swarming/w/ir/cache/builder/src/out/bad_devices.json I 3.113s Main kill 15 10087 (adb [adb -L tcp:5037 fork-server server --reply-fd 12]) I 3.114s Main kill 15 10110 (adb [/b/swarming/w/ir/cache/builder/src/third_party/android_tools/sdk/platform-tools/adb -s 00b1e3f189a332e5 logcat -v threadtime]) I 3.118s Main kill 3 10110 (adb []) I 3.120s Main kill 9 10110 (adb []) E 3.122s Main Unable to kill 10110 (adb []) W 6.127s Main USB reset disabled for 01ff3a86c26b4d26 (crbug.com/642914) W 6.127s Main USB reset disabled for 00c56ac971923a54 (crbug.com/642914) W 6.128s Main USB reset disabled for 01e1be00ca881247 (crbug.com/642914) E 36.180s RecoverDevice(00c56ac971923a54) Failure while waiting for 00c56ac971923a54. Attempting to recover. Traceback (most recent call last): File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py", line 71, in RecoverDevice device.WaitUntilFullyBooted(retries=0) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 196, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 168, in _JoinAll thread.ReraiseIfException() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 91, in run self._ret = self._func(*self._args, **self._kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/device_utils.py", line 815, in WaitUntilFullyBooted self.adb.WaitForDevice() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 866, in WaitForDevice self._RunDeviceAdbCmd(['wait-for-device'], timeout, retries) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 315, in _RunDeviceAdbCmd ensure_logs_on_timeout=ensure_logs_on_timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 273, in _RunAdbCmd timeout, env=cls._ADB_ENV) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/cmd_helper.py", line 419, in GetCmdStatusAndOutputWithTimeout raise TimeoutError(output.getvalue()) CommandTimeoutError E 36.181s RecoverDevice(01ff3a86c26b4d26) Failure while waiting for 01ff3a86c26b4d26. Attempting to recover. Traceback (most recent call last): File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py", line 71, in RecoverDevice device.WaitUntilFullyBooted(retries=0) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 196, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 168, in _JoinAll thread.ReraiseIfException() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 91, in run self._ret = self._func(*self._args, **self._kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/device_utils.py", line 815, in WaitUntilFullyBooted self.adb.WaitForDevice() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 866, in WaitForDevice self._RunDeviceAdbCmd(['wait-for-device'], timeout, retries) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 315, in _RunDeviceAdbCmd ensure_logs_on_timeout=ensure_logs_on_timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 273, in _RunAdbCmd timeout, env=cls._ADB_ENV) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/cmd_helper.py", line 419, in GetCmdStatusAndOutputWithTimeout raise TimeoutError(output.getvalue()) CommandTimeoutError E 36.184s RecoverDevice(01e1be00ca881247) Failure while waiting for 01e1be00ca881247. Attempting to recover. Traceback (most recent call last): File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py", line 71, in RecoverDevice device.WaitUntilFullyBooted(retries=0) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 196, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 168, in _JoinAll thread.ReraiseIfException() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 91, in run self._ret = self._func(*self._args, **self._kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/device_utils.py", line 815, in WaitUntilFullyBooted self.adb.WaitForDevice() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 866, in WaitForDevice self._RunDeviceAdbCmd(['wait-for-device'], timeout, retries) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 315, in _RunDeviceAdbCmd ensure_logs_on_timeout=ensure_logs_on_timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 273, in _RunAdbCmd timeout, env=cls._ADB_ENV) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/cmd_helper.py", line 419, in GetCmdStatusAndOutputWithTimeout raise TimeoutError(output.getvalue()) CommandTimeoutError E 36.190s RecoverDevice(00c56ac971923a54) Failed to reboot 00c56ac971923a54. Traceback (most recent call last): File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py", line 79, in RecoverDevice device.Reboot(block=False, timeout=5, retries=0) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 196, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 168, in _JoinAll thread.ReraiseIfException() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 91, in run self._ret = self._func(*self._args, **self._kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/device_utils.py", line 843, in Reboot self.adb.Reboot() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 917, in Reboot self._RunDeviceAdbCmd(cmd, timeout, retries) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 315, in _RunDeviceAdbCmd ensure_logs_on_timeout=ensure_logs_on_timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 289, in _RunAdbCmd raise device_errors.DeviceUnreachableError(device_serial) DeviceUnreachableError: 00c56ac971923a54 E 36.190s RecoverDevice(01e1be00ca881247) Failed to reboot 01e1be00ca881247. Traceback (most recent call last): File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py", line 79, in RecoverDevice device.Reboot(block=False, timeout=5, retries=0) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 196, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 168, in _JoinAll thread.ReraiseIfException() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 91, in run self._ret = self._func(*self._args, **self._kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/device_utils.py", line 843, in Reboot self.adb.Reboot() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 917, in Reboot self._RunDeviceAdbCmd(cmd, timeout, retries) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 315, in _RunDeviceAdbCmd ensure_logs_on_timeout=ensure_logs_on_timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 289, in _RunAdbCmd raise device_errors.DeviceUnreachableError(device_serial) DeviceUnreachableError: 01e1be00ca881247 I 36.192s RecoverDevice(00c56ac971923a54) Adding 00c56ac971923a54 to blacklist /b/swarming/w/ir/cache/builder/src/out/bad_devices.json for reason: reboot_failure I 36.192s RecoverDevice(01e1be00ca881247) Adding 01e1be00ca881247 to blacklist /b/swarming/w/ir/cache/builder/src/out/bad_devices.json for reason: reboot_failure E 36.195s RecoverDevice(01ff3a86c26b4d26) Failed to reboot 01ff3a86c26b4d26. Traceback (most recent call last): File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py", line 79, in RecoverDevice device.Reboot(block=False, timeout=5, retries=0) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 196, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 168, in _JoinAll thread.ReraiseIfException() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 91, in run self._ret = self._func(*self._args, **self._kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/device_utils.py", line 843, in Reboot self.adb.Reboot() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 917, in Reboot self._RunDeviceAdbCmd(cmd, timeout, retries) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 315, in _RunDeviceAdbCmd ensure_logs_on_timeout=ensure_logs_on_timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper return impl() File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl return f(*args, **kwargs) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 289, in _RunAdbCmd raise device_errors.DeviceUnreachableError(device_serial) DeviceUnreachableError: 01ff3a86c26b4d26 I 36.195s RecoverDevice(01ff3a86c26b4d26) Adding 01ff3a86c26b4d26 to blacklist /b/swarming/w/ir/cache/builder/src/out/bad_devices.json for reason: reboot_failure I 96.270s RecoverDevice(00c56ac971923a54) Still working on WaitUntilFullyBooted(00c56ac971923a54, retries=0, timeout=300) I 96.271s RecoverDevice(01e1be00ca881247) Still working on WaitUntilFullyBooted(01e1be00ca881247, retries=0, timeout=300) I 96.274s RecoverDevice(01ff3a86c26b4d26) Still working on WaitUntilFullyBooted(01ff3a86c26b4d26, retries=0, timeout=300) I 156.344s RecoverDevice(01ff3a86c26b4d26) Still working on WaitUntilFullyBooted(01ff3a86c26b4d26, retries=0, timeout=300) I 156.344s RecoverDevice(01e1be00ca881247) Still working on WaitUntilFullyBooted(01e1be00ca881247, retries=0, timeout=300) I 156.346s RecoverDevice(00c56ac971923a54) Still working on WaitUntilFullyBooted(00c56ac971923a54, retries=0, timeout=300) I 216.417s RecoverDevice(01e1be00ca881247) Still working on WaitUntilFullyBooted(01e1be00ca881247, retries=0, timeout=300) I 216.417s RecoverDevice(01ff3a86c26b4d26) Still working on WaitUntilFullyBooted(01ff3a86c26b4d26, retries=0, timeout=300) I 216.417s RecoverDevice(00c56ac971923a54) Still working on WaitUntilFullyBooted(00c56ac971923a54, retries=0, timeout=300) I 276.491s RecoverDevice(00c56ac971923a54) Still working on WaitUntilFullyBooted(00c56ac971923a54, retries=0, timeout=300) I 276.491s RecoverDevice(01e1be00ca881247) Still working on WaitUntilFullyBooted(01e1be00ca881247, retries=0, timeout=300) I 276.491s RecoverDevice(01ff3a86c26b4d26) Still working on WaitUntilFullyBooted(01ff3a86c26b4d26, retries=0, timeout=300) C 336.265s RecoverDevice(01ff3a86c26b4d26) Timed out. Dumping threads. C 336.266s RecoverDevice(01e1be00ca881247) Timed out. Dumping threads. C 336.266s RecoverDevice(00c56ac971923a54) Timed out. Dumping threads. C 336.266s RecoverDevice(01ff3a86c26b4d26) ******************************************************************************** C 336.266s RecoverDevice(01ff3a86c26b4d26) Stack dump for thread 'TimeoutThread-1-for-RecoverDevice(01ff3a86c26b4d26)' C 336.266s RecoverDevice(00c56ac971923a54) ******************************************************************************** C 336.267s RecoverDevice(01ff3a86c26b4d26) ******************************************************************************** C 336.267s RecoverDevice(01e1be00ca881247) ******************************************************************************** C 336.267s RecoverDevice(00c56ac971923a54) Stack dump for thread 'TimeoutThread-1-for-RecoverDevice(00c56ac971923a54)' C 336.267s RecoverDevice(01e1be00ca881247) Stack dump for thread 'TimeoutThread-1-for-RecoverDevice(01e1be00ca881247)' C 336.267s RecoverDevice(00c56ac971923a54) ******************************************************************************** C 336.267s RecoverDevice(01e1be00ca881247) ******************************************************************************** C 336.269s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cipd_bin_packages/lib/python2.7/threading.py", line 774, in __bootstrap C 336.269s RecoverDevice(01ff3a86c26b4d26) self.__bootstrap_inner() C 336.270s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cipd_bin_packages/lib/python2.7/threading.py", line 801, in __bootstrap_inner C 336.270s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cipd_bin_packages/lib/python2.7/threading.py", line 774, in __bootstrap C 336.270s RecoverDevice(01ff3a86c26b4d26) self.run() C 336.271s RecoverDevice(00c56ac971923a54) self.__bootstrap_inner() C 336.271s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 91, in run C 336.271s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cipd_bin_packages/lib/python2.7/threading.py", line 774, in __bootstrap C 336.271s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cipd_bin_packages/lib/python2.7/threading.py", line 801, in __bootstrap_inner C 336.271s RecoverDevice(01ff3a86c26b4d26) self._ret = self._func(*self._args, **self._kwargs) C 336.271s RecoverDevice(01e1be00ca881247) self.__bootstrap_inner() C 336.271s RecoverDevice(00c56ac971923a54) self.run() C 336.272s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> C 336.272s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cipd_bin_packages/lib/python2.7/threading.py", line 801, in __bootstrap_inner C 336.272s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 91, in run C 336.272s RecoverDevice(01ff3a86c26b4d26) child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), C 336.272s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl C 336.272s RecoverDevice(00c56ac971923a54) self._ret = self._func(*self._args, **self._kwargs) C 336.272s RecoverDevice(01ff3a86c26b4d26) return f(*args, **kwargs) C 336.273s RecoverDevice(01e1be00ca881247) self.run() C 336.273s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> C 336.273s RecoverDevice(00c56ac971923a54) child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), C 336.273s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 91, in run C 336.273s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl C 336.273s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/device_utils.py", line 815, in WaitUntilFullyBooted C 336.273s RecoverDevice(01e1be00ca881247) self._ret = self._func(*self._args, **self._kwargs) C 336.273s RecoverDevice(00c56ac971923a54) return f(*args, **kwargs) C 336.274s RecoverDevice(01ff3a86c26b4d26) self.adb.WaitForDevice() C 336.274s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 151, in <lambda> C 336.274s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/device_utils.py", line 815, in WaitUntilFullyBooted C 336.274s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 866, in WaitForDevice C 336.274s RecoverDevice(01e1be00ca881247) child_thread = reraiser_thread.ReraiserThread(lambda: func(*args, **kwargs), C 336.274s RecoverDevice(00c56ac971923a54) self.adb.WaitForDevice() C 336.274s RecoverDevice(01ff3a86c26b4d26) self._RunDeviceAdbCmd(['wait-for-device'], timeout, retries) C 336.274s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl C 336.274s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 866, in WaitForDevice C 336.275s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 315, in _RunDeviceAdbCmd C 336.275s RecoverDevice(01e1be00ca881247) return f(*args, **kwargs) C 336.275s RecoverDevice(00c56ac971923a54) self._RunDeviceAdbCmd(['wait-for-device'], timeout, retries) C 336.275s RecoverDevice(01ff3a86c26b4d26) ensure_logs_on_timeout=ensure_logs_on_timeout) C 336.275s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/device_utils.py", line 815, in WaitUntilFullyBooted C 336.275s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 315, in _RunDeviceAdbCmd C 336.275s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper C 336.275s RecoverDevice(01e1be00ca881247) self.adb.WaitForDevice() C 336.275s RecoverDevice(00c56ac971923a54) ensure_logs_on_timeout=ensure_logs_on_timeout) C 336.275s RecoverDevice(01ff3a86c26b4d26) return impl() C 336.276s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 866, in WaitForDevice C 336.276s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper C 336.276s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl C 336.276s RecoverDevice(01e1be00ca881247) self._RunDeviceAdbCmd(['wait-for-device'], timeout, retries) C 336.276s RecoverDevice(00c56ac971923a54) return impl() C 336.276s RecoverDevice(01ff3a86c26b4d26) return f(*args, **kwargs) C 336.276s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 315, in _RunDeviceAdbCmd C 336.276s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl C 336.277s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 273, in _RunAdbCmd C 336.277s RecoverDevice(01e1be00ca881247) ensure_logs_on_timeout=ensure_logs_on_timeout) C 336.277s RecoverDevice(00c56ac971923a54) return f(*args, **kwargs) C 336.277s RecoverDevice(01ff3a86c26b4d26) timeout, env=cls._ADB_ENV) C 336.277s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 51, in timeout_retry_wrapper C 336.277s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 273, in _RunAdbCmd C 336.277s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/cmd_helper.py", line 414, in GetCmdStatusAndOutputWithTimeout C 336.277s RecoverDevice(01e1be00ca881247) return impl() C 336.277s RecoverDevice(00c56ac971923a54) timeout, env=cls._ADB_ENV) C 336.277s RecoverDevice(01ff3a86c26b4d26) for data in _IterProcessStdout(process, timeout=timeout): C 336.278s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 47, in impl C 336.278s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/cmd_helper.py", line 414, in GetCmdStatusAndOutputWithTimeout C 336.278s RecoverDevice(01ff3a86c26b4d26) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/cmd_helper.py", line 276, in _IterProcessStdoutFcntl C 336.278s RecoverDevice(01e1be00ca881247) return f(*args, **kwargs) C 336.278s RecoverDevice(00c56ac971923a54) for data in _IterProcessStdout(process, timeout=timeout): C 336.278s RecoverDevice(01ff3a86c26b4d26) [child_fd], [], [], iter_aware_poll_interval) C 336.278s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/sdk/adb_wrapper.py", line 273, in _RunAdbCmd C 336.278s RecoverDevice(00c56ac971923a54) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/cmd_helper.py", line 276, in _IterProcessStdoutFcntl C 336.278s RecoverDevice(01ff3a86c26b4d26) ******************************************************************************** C 336.279s RecoverDevice(01e1be00ca881247) timeout, env=cls._ADB_ENV) C 336.279s RecoverDevice(00c56ac971923a54) [child_fd], [], [], iter_aware_poll_interval) E 336.279s RecoverDevice(01ff3a86c26b4d26) Timed out while waiting for 01ff3a86c26b4d26. Traceback (most recent call last): File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py", line 107, in RecoverDevice retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 196, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 161, in _JoinAll (len(alive_threads), len(self._threads))) CommandTimeoutError: Timed out waiting for 1 of 1 threads. C 336.279s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/cmd_helper.py", line 414, in GetCmdStatusAndOutputWithTimeout C 336.279s RecoverDevice(00c56ac971923a54) ******************************************************************************** I 336.279s RecoverDevice(01ff3a86c26b4d26) Adding 01ff3a86c26b4d26 to blacklist /b/swarming/w/ir/cache/builder/src/out/bad_devices.json for reason: reboot_timeout C 336.280s RecoverDevice(01e1be00ca881247) for data in _IterProcessStdout(process, timeout=timeout): E 336.280s RecoverDevice(00c56ac971923a54) Timed out while waiting for 00c56ac971923a54. Traceback (most recent call last): File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py", line 107, in RecoverDevice retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 196, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 161, in _JoinAll (len(alive_threads), len(self._threads))) CommandTimeoutError: Timed out waiting for 1 of 1 threads. I 336.280s RecoverDevice(00c56ac971923a54) Adding 00c56ac971923a54 to blacklist /b/swarming/w/ir/cache/builder/src/out/bad_devices.json for reason: reboot_timeout C 336.281s RecoverDevice(01e1be00ca881247) File: "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/cmd_helper.py", line 276, in _IterProcessStdoutFcntl C 336.281s RecoverDevice(01e1be00ca881247) [child_fd], [], [], iter_aware_poll_interval) C 336.281s RecoverDevice(01e1be00ca881247) ******************************************************************************** E 336.281s RecoverDevice(01e1be00ca881247) Timed out while waiting for 01e1be00ca881247. Traceback (most recent call last): File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/tools/device_recovery.py", line 107, in RecoverDevice retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/android/decorators.py", line 57, in timeout_retry_wrapper retry_if_func=retry_if_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/timeout_retry.py", line 158, in Run error_log_func=error_log_func) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 196, in JoinAll self._JoinAll(watcher, timeout) File "/b/swarming/w/ir/cache/builder/src/third_party/catapult/devil/devil/utils/reraiser_thread.py", line 161, in _JoinAll (len(alive_threads), len(self._threads))) CommandTimeoutError: Timed out waiting for 1 of 1 threads. I 336.281s RecoverDevice(01e1be00ca881247) Adding 01e1be00ca881247 to blacklist /b/swarming/w/ir/cache/builder/src/out/bad_devices.json for reason: reboot_timeout