test/Android.run-test.mk

   1 # Copyright (C) 2011 The Android Open Source Project
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #      http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 #
  15
  16 LOCAL_PATH := $(call my-dir)
  17
  18 include art/build/Android.common_test.mk
  19
  20 # List of all tests of the form 003-omnibus-opcodes.
  21 TEST_ART_RUN_TESTS := $(wildcard $(LOCAL_PATH)/[0-9]*)
  22 TEST_ART_RUN_TESTS := $(subst $(LOCAL_PATH)/,, $(TEST_ART_RUN_TESTS))
  23
  24 ########################################################################
  25 # The art-run-tests module, used to build all run-tests into an image.
  26
  27 # The path where build only targets will be output, e.g.
  28 # out/target/product/generic_x86_64/obj/PACKAGING/art-run-tests_intermediates/DATA
  29 art_run_tests_dir := $(call intermediates-dir-for,PACKAGING,art-run-tests)/DATA
  30
  31 # A generated list of prerequisites that call 'run-test --build-only', the actual prerequisite is
  32 # an empty file touched in the intermediate directory.
  33 TEST_ART_RUN_TEST_BUILD_RULES :=
  34
  35 # Dependencies for actually running a run-test.
  36 TEST_ART_RUN_TEST_DEPENDENCIES := \
  37   $(DX) \
  38   $(HOST_OUT_EXECUTABLES)/jasmin \
  39   $(HOST_OUT_EXECUTABLES)/smali \
  40   $(HOST_OUT_EXECUTABLES)/dexmerger \
  41   $(JACK)
  42
  43 TEST_ART_RUN_TEST_ORDERONLY_DEPENDENCIES := setup-jack-server
  44
  45 ifeq ($(ART_TEST_DEBUG_GC),true)
  46   ART_TEST_WITH_STRACE := true
  47 endif
  48
  49 # Helper to create individual build targets for tests. Must be called with $(eval).
  50 # $(1): the test number
  51 define define-build-art-run-test
  52   dmart_target := $(art_run_tests_dir)/art-run-tests/$(1)/touch
  53   run_test_options = --build-only
  54   ifeq ($(ART_TEST_QUIET),true)
  55     run_test_options += --quiet
  56   endif
  57 $$(dmart_target): PRIVATE_RUN_TEST_OPTIONS := $$(run_test_options)
  58 $$(dmart_target): $(TEST_ART_RUN_TEST_DEPENDENCIES) $(TARGET_JACK_CLASSPATH_DEPENDENCIES) | $(TEST_ART_RUN_TEST_ORDERONLY_DEPENDENCIES)
  59         $(hide) rm -rf $$(dir $$@) && mkdir -p $$(dir $$@)
  60         $(hide) DX=$(abspath $(DX)) JASMIN=$(abspath $(HOST_OUT_EXECUTABLES)/jasmin) \
  61           SMALI=$(abspath $(HOST_OUT_EXECUTABLES)/smali) \
  62           DXMERGER=$(abspath $(HOST_OUT_EXECUTABLES)/dexmerger) \
  63           JACK_VERSION=$(JACK_DEFAULT_VERSION) \
  64           JACK=$(abspath $(JACK)) \
  65           JACK_VERSION=$(JACK_DEFAULT_VERSION) \
  66           JACK_CLASSPATH=$(TARGET_JACK_CLASSPATH) \
  67           $(LOCAL_PATH)/run-test $$(PRIVATE_RUN_TEST_OPTIONS) --output-path $$(abspath $$(dir $$@)) $(1)
  68         $(hide) touch $$@
  69
  70   TEST_ART_RUN_TEST_BUILD_RULES += $$(dmart_target)
  71   dmart_target :=
  72   run_test_options :=
  73 endef
  74 $(foreach test, $(TEST_ART_RUN_TESTS), $(eval $(call define-build-art-run-test,$(test))))
  75
  76 include $(CLEAR_VARS)
  77 LOCAL_MODULE_TAGS := tests
  78 LOCAL_MODULE := art-run-tests
  79 LOCAL_ADDITIONAL_DEPENDENCIES := $(TEST_ART_RUN_TEST_BUILD_RULES)
  80 # The build system use this flag to pick up files generated by declare-make-art-run-test.
  81 LOCAL_PICKUP_FILES := $(art_run_tests_dir)
  82
  83 include $(BUILD_PHONY_PACKAGE)
  84
  85 # Clear temp vars.
  86 art_run_tests_dir :=
  87 define-build-art-run-test :=
  88 TEST_ART_RUN_TEST_BUILD_RULES :=
  89
  90 ########################################################################
  91 # General rules to build and run a run-test.
  92
  93 TARGET_TYPES := host target
  94 PREBUILD_TYPES :=
  95 ifeq ($(ART_TEST_RUN_TEST_PREBUILD),true)
  96   PREBUILD_TYPES += prebuild
  97 endif
  98 ifeq ($(ART_TEST_RUN_TEST_NO_PREBUILD),true)
  99   PREBUILD_TYPES += no-prebuild
 100 endif
 101 ifeq ($(ART_TEST_RUN_TEST_NO_DEX2OAT),true)
 102   PREBUILD_TYPES += no-dex2oat
 103 endif
 104 COMPILER_TYPES :=
 105 ifeq ($(ART_TEST_INTERPRETER_ACCESS_CHECKS),true)
 106   COMPILER_TYPES += interp-ac
 107 endif
 108 ifeq ($(ART_TEST_INTERPRETER),true)
 109   COMPILER_TYPES += interpreter
 110 endif
 111 ifeq ($(ART_TEST_JIT),true)
 112   COMPILER_TYPES += jit
 113 endif
 114 ifeq ($(ART_TEST_OPTIMIZING),true)
 115   COMPILER_TYPES += optimizing
 116 endif
 117 RELOCATE_TYPES := relocate
 118 ifeq ($(ART_TEST_RUN_TEST_NO_RELOCATE),true)
 119   RELOCATE_TYPES += no-relocate
 120 endif
 121 ifeq ($(ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT),true)
 122   RELOCATE_TYPES += relocate-npatchoat
 123 endif
 124 TRACE_TYPES := ntrace
 125 ifeq ($(ART_TEST_TRACE),true)
 126   TRACE_TYPES += trace
 127 endif
 128 ifeq ($(ART_TEST_TRACE_STREAM),true)
 129   TRACE_TYPES += stream
 130 endif
 131 GC_TYPES := cms
 132 ifeq ($(ART_TEST_GC_STRESS),true)
 133   GC_TYPES += gcstress
 134 endif
 135 ifeq ($(ART_TEST_GC_VERIFY),true)
 136   GC_TYPES += gcverify
 137 endif
 138 JNI_TYPES := checkjni
 139 ifeq ($(ART_TEST_JNI_FORCECOPY),true)
 140   JNI_TYPES += forcecopy
 141 endif
 142 IMAGE_TYPES := image
 143 ifeq ($(ART_TEST_RUN_TEST_NO_IMAGE),true)
 144   IMAGE_TYPES += no-image
 145 endif
 146 ifeq ($(ART_TEST_RUN_TEST_MULTI_IMAGE),true)
 147   IMAGE_TYPES := multiimage
 148 endif
 149 ifeq ($(ART_TEST_PIC_IMAGE),true)
 150   IMAGE_TYPES += picimage
 151   ifeq ($(ART_TEST_RUN_TEST_MULTI_IMAGE),true)
 152     IMAGE_TYPES := multipicimage
 153   endif
 154 endif
 155 PICTEST_TYPES := npictest
 156 ifeq ($(ART_TEST_PIC_TEST),true)
 157   PICTEST_TYPES += pictest
 158 endif
 159 RUN_TYPES :=
 160 ifeq ($(ART_TEST_RUN_TEST_DEBUG),true)
 161   RUN_TYPES += debug
 162 endif
 163 ifeq ($(ART_TEST_RUN_TEST_NDEBUG),true)
 164   RUN_TYPES += ndebug
 165 endif
 166 DEBUGGABLE_TYPES := ndebuggable
 167 ifeq ($(ART_TEST_RUN_TEST_DEBUGGABLE),true)
 168 DEBUGGABLE_TYPES += debuggable
 169 endif
 170 ADDRESS_SIZES_TARGET := $(ART_PHONY_TEST_TARGET_SUFFIX)
 171 ADDRESS_SIZES_HOST := $(ART_PHONY_TEST_HOST_SUFFIX)
 172 ifeq ($(ART_TEST_RUN_TEST_2ND_ARCH),true)
 173   ADDRESS_SIZES_TARGET += $(2ND_ART_PHONY_TEST_TARGET_SUFFIX)
 174   ADDRESS_SIZES_HOST += $(2ND_ART_PHONY_TEST_HOST_SUFFIX)
 175 endif
 176 ALL_ADDRESS_SIZES := 64 32
 177
 178 # List all run test names with number arguments agreeing with the comment above.
 179 define all-run-test-names
 180   $(foreach target, $(1), \
 181     $(foreach run-type, $(2), \
 182       $(foreach prebuild, $(3), \
 183         $(foreach compiler, $(4), \
 184           $(foreach relocate, $(5), \
 185             $(foreach trace, $(6), \
 186               $(foreach gc, $(7), \
 187                 $(foreach jni, $(8), \
 188                   $(foreach image, $(9), \
 189                     $(foreach pictest, $(10), \
 190                       $(foreach debuggable, $(11), \
 191                         $(foreach test, $(12), \
 192                           $(foreach address_size, $(13), \
 193                             test-art-$(target)-run-test-$(run-type)-$(prebuild)-$(compiler)-$(relocate)-$(trace)-$(gc)-$(jni)-$(image)-$(pictest)-$(debuggable)-$(test)$(address_size) \
 194                     )))))))))))))
 195 endef  # all-run-test-names
 196
 197 # To generate a full list or tests:
 198 # $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES),$(COMPILER_TYPES), \
 199 #        $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \
 200 #        $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 201
 202 # Convert's a rule name to the form used in variables, e.g. no-relocate to NO_RELOCATE
 203 define name-to-var
 204 $(shell echo $(1) | tr '[:lower:]' '[:upper:]' | tr '-' '_')
 205 endef  # name-to-var
 206
 207 ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 208         $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 209         $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(ART_TEST_RUN_TEST_SKIP), $(ALL_ADDRESS_SIZES))
 210
 211
 212 # Disable 137-cfi (b/27391690).
 213 # Disable 577-profile-foreign-dex (b/27454772).
 214 TEST_ART_BROKEN_ALL_TARGET_TESTS := \
 215   577-profile-foreign-dex \
 216
 217 ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 218     $(COMPILER_TYPES), $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 219     $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_ALL_TARGET_TESTS), \
 220     $(ALL_ADDRESS_SIZES))
 221
 222 TEST_ART_BROKEN_ALL_TARGET_TESTS :=
 223
 224 # Tests that are timing sensitive and flaky on heavily loaded systems.
 225 TEST_ART_TIMING_SENSITIVE_RUN_TESTS := \
 226   002-sleep \
 227   053-wait-some \
 228   055-enum-performance \
 229   133-static-invoke-super
 230
 231 # disable timing sensitive tests on "dist" builds.
 232 ifdef dist_goal
 233   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 234         $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 235         $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_TIMING_SENSITIVE_RUN_TESTS), $(ALL_ADDRESS_SIZES))
 236 endif
 237
 238 # 147-stripped-dex-fallback isn't supported on device because --strip-dex
 239 # requires the zip command.
 240 # 569-checker-pattern-replacement tests behaviour present only on host.
 241 TEST_ART_BROKEN_TARGET_TESTS := \
 242   147-stripped-dex-fallback \
 243   569-checker-pattern-replacement
 244
 245 ifneq (,$(filter target,$(TARGET_TYPES)))
 246   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \
 247       $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 248       $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_TARGET_TESTS), $(ALL_ADDRESS_SIZES))
 249 endif
 250
 251 TEST_ART_BROKEN_TARGET_TESTS :=
 252
 253 # Tests that require python3.
 254 TEST_ART_PYTHON3_DEPENDENCY_RUN_TESTS := \
 255   960-default-smali \
 256   961-default-iface-resolution-generated \
 257   964-default-iface-init-generated \
 258   968-default-partial-compile-generated \
 259   969-iface-super \
 260   970-iface-super-resolution-generated \
 261   971-iface-super
 262
 263 # Check if we have python3 to run our tests.
 264 ifeq ($(wildcard /usr/bin/python3),)
 265   $(warning "No python3 found. Disabling tests: $(TEST_ART_PYTHON3_DEPENDENCY_RUN_TESTS)")
 266
 267   # Currently disable tests requiring python3 when it is not installed.
 268   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 269         $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 270         $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_PYTHON3_DEPENDENCY_RUN_TESTS), $(ALL_ADDRESS_SIZES))
 271 endif
 272
 273 TEST_ART_TIMING_SENSITIVE_RUN_TESTS :=
 274
 275 # Note 116-nodex2oat is not broken per-se it just doesn't (and isn't meant to) work with --prebuild.
 276 TEST_ART_BROKEN_PREBUILD_RUN_TESTS := \
 277   116-nodex2oat \
 278   118-noimage-dex2oat \
 279   134-nodex2oat-nofallback
 280
 281 ifneq (,$(filter prebuild,$(PREBUILD_TYPES)))
 282   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),prebuild, \
 283       $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 284       $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_PREBUILD_RUN_TESTS), $(ALL_ADDRESS_SIZES))
 285 endif
 286
 287 TEST_ART_BROKEN_PREBUILD_RUN_TESTS :=
 288
 289 # 554-jit-profile-file is disabled because it needs a primary oat file to know what it should save.
 290 # 529 and 555: b/27784033
 291 TEST_ART_BROKEN_NO_PREBUILD_TESTS := \
 292   117-nopatchoat \
 293   147-stripped-dex-fallback \
 294   554-jit-profile-file \
 295   529-checker-unresolved \
 296   555-checker-regression-x86const \
 297   608-checker-unresolved-lse
 298
 299 ifneq (,$(filter no-prebuild,$(PREBUILD_TYPES)))
 300   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),no-prebuild, \
 301       $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 302       $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_NO_PREBUILD_TESTS), $(ALL_ADDRESS_SIZES))
 303 endif
 304
 305 TEST_ART_BROKEN_NO_PREBUILD_TESTS :=
 306
 307 # Note 117-nopatchoat is not broken per-se it just doesn't work (and isn't meant to) without
 308 # --prebuild --relocate
 309 TEST_ART_BROKEN_NO_RELOCATE_TESTS := \
 310   117-nopatchoat \
 311   118-noimage-dex2oat \
 312   119-noimage-patchoat \
 313   554-jit-profile-file
 314
 315 ifneq (,$(filter no-relocate,$(RELOCATE_TYPES)))
 316   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 317       $(COMPILER_TYPES), no-relocate,$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 318       $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_NO_RELOCATE_TESTS), $(ALL_ADDRESS_SIZES))
 319 endif
 320
 321 TEST_ART_BROKEN_NO_RELOCATE_TESTS :=
 322
 323 # Temporarily disable some broken tests when forcing access checks in interpreter b/22414682
 324 TEST_ART_BROKEN_INTERPRETER_ACCESS_CHECK_TESTS := \
 325   137-cfi
 326
 327 ifneq (,$(filter interp-ac,$(COMPILER_TYPES)))
 328   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 329       interp-ac,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 330       $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_INTERPRETER_ACCESS_CHECK_TESTS), $(ALL_ADDRESS_SIZES))
 331 endif
 332
 333 TEST_ART_BROKEN_INTERPRETER_ACCESS_CHECK_TESTS :=
 334
 335 # Tests that are broken with GC stress.
 336 # * 137-cfi needs to unwind a second forked process. We're using a primitive sleep to wait till we
 337 #   hope the second process got into the expected state. The slowness of gcstress makes this bad.
 338 # * 961-default-iface-resolution-generated and 964-default-iface-init-generated are very long tests
 339 #   that often will take more than the timeout to run when gcstress is enabled. This is because
 340 #   gcstress slows down allocations significantly which these tests do a lot.
 341 TEST_ART_BROKEN_GCSTRESS_RUN_TESTS := \
 342   137-cfi \
 343   961-default-iface-resolution-generated \
 344   964-default-iface-init-generated
 345
 346 ifneq (,$(filter gcstress,$(GC_TYPES)))
 347   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 348       $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),gcstress,$(JNI_TYPES), \
 349       $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_GCSTRESS_RUN_TESTS), $(ALL_ADDRESS_SIZES))
 350 endif
 351
 352 TEST_ART_BROKEN_GCSTRESS_RUN_TESTS :=
 353
 354 # 115-native-bridge setup is complicated. Need to implement it correctly for the target.
 355 ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES),$(COMPILER_TYPES), \
 356     $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), 115-native-bridge, \
 357     $(ALL_ADDRESS_SIZES))
 358
 359 # 130-hprof dumps the heap and runs hprof-conv to check whether the file is somewhat readable. This
 360 # is only possible on the host.
 361 # TODO: Turn off all the other combinations, this is more about testing actual ART code. A gtest is
 362 #       very hard to write here, as (for a complete test) JDWP must be set up.
 363 ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \
 364     $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \
 365     $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),130-hprof,$(ALL_ADDRESS_SIZES))
 366
 367 # 131 is an old test. The functionality has been implemented at an earlier stage and is checked
 368 # in tests 138. Blacklisted for debug builds since these builds have duplicate classes checks which
 369 # punt to interpreter.
 370 ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),debug,$(PREBUILD_TYPES), \
 371     $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \
 372     $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),131-structural-change,$(ALL_ADDRESS_SIZES))
 373
 374 # 138-duplicate-classes-check. Turned on for debug builds since debug builds have duplicate classes
 375 # checks enabled, b/2133391.
 376 ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),ndebug,$(PREBUILD_TYPES), \
 377     $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \
 378     $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),138-duplicate-classes-check,$(ALL_ADDRESS_SIZES))
 379
 380 # All these tests check that we have sane behavior if we don't have a patchoat or dex2oat.
 381 # Therefore we shouldn't run them in situations where we actually don't have these since they
 382 # explicitly test for them. These all also assume we have an image.
 383 # 147-stripped-dex-fallback is disabled because it requires --prebuild.
 384 # 554-jit-profile-file is disabled because it needs a primary oat file to know what it should save.
 385 TEST_ART_BROKEN_FALLBACK_RUN_TESTS := \
 386   116-nodex2oat \
 387   117-nopatchoat \
 388   118-noimage-dex2oat \
 389   119-noimage-patchoat \
 390   137-cfi \
 391   138-duplicate-classes-check2 \
 392   147-stripped-dex-fallback \
 393   554-jit-profile-file
 394
 395 # This test fails without an image.
 396 TEST_ART_BROKEN_NO_IMAGE_RUN_TESTS := \
 397   137-cfi \
 398   138-duplicate-classes-check
 399
 400 ifneq (,$(filter no-dex2oat,$(PREBUILD_TYPES)))
 401   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),no-dex2oat, \
 402       $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \
 403       $(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_FALLBACK_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 404 endif
 405
 406
 407 ifneq (,$(filter no-image,$(IMAGE_TYPES)))
 408   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 409       $(COMPILER_TYPES), $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),no-image, \
 410       $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_FALLBACK_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 411   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 412       $(COMPILER_TYPES), $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),no-image, \
 413       $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_NO_IMAGE_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 414 endif
 415
 416 ifneq (,$(filter relocate-npatchoat,$(RELOCATE_TYPES)))
 417   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 418       $(COMPILER_TYPES), relocate-npatchoat,$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 419       $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_FALLBACK_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 420 endif
 421
 422 TEST_ART_BROKEN_FALLBACK_RUN_TESTS :=
 423
 424 # 137:
 425 # This test unrolls and expects managed frames, but tracing means we run the interpreter.
 426 # 802 and 570-checker-osr:
 427 # This test dynamically enables tracing to force a deoptimization. This makes the test meaningless
 428 # when already tracing, and writes an error message that we do not want to check for.
 429 TEST_ART_BROKEN_TRACING_RUN_TESTS := \
 430   087-gc-after-link \
 431   137-cfi \
 432   141-class-unload \
 433   570-checker-osr \
 434   802-deoptimization
 435
 436 ifneq (,$(filter trace stream,$(TRACE_TYPES)))
 437   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 438       $(COMPILER_TYPES),$(RELOCATE_TYPES),trace stream,$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \
 439       $(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_TRACING_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 440 endif
 441
 442 # Known broken tests for the interpreter.
 443 # CFI unwinding expects managed frames.
 444 TEST_ART_BROKEN_INTERPRETER_RUN_TESTS := \
 445   137-cfi \
 446   554-jit-profile-file
 447
 448 ifneq (,$(filter interpreter,$(COMPILER_TYPES)))
 449   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 450       interpreter,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 451       $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_INTERPRETER_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 452 endif
 453
 454 TEST_ART_BROKEN_INTERPRETER_RUN_TESTS :=
 455
 456 # Known broken tests for the JIT.
 457 # CFI unwinding expects managed frames, and the test does not iterate enough to even compile. JIT
 458 # also uses Generic JNI instead of the JNI compiler.
 459 TEST_ART_BROKEN_JIT_RUN_TESTS := \
 460   137-cfi
 461
 462 ifneq (,$(filter jit,$(COMPILER_TYPES)))
 463   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 464       jit,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 465       $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_JIT_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 466 endif
 467
 468 TEST_ART_BROKEN_JIT_RUN_TESTS :=
 469
 470 # Known broken tests for the mips32 optimizing compiler backend.
 471 TEST_ART_BROKEN_OPTIMIZING_MIPS_RUN_TESTS := \
 472     510-checker-try-catch \
 473
 474 ifeq (mips,$(TARGET_ARCH))
 475   ifneq (,$(filter optimizing,$(COMPILER_TYPES)))
 476     ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \
 477         optimizing,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 478         $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \
 479         $(TEST_ART_BROKEN_OPTIMIZING_MIPS_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 480   endif
 481 endif
 482
 483 TEST_ART_BROKEN_OPTIMIZING_MIPS_RUN_TESTS :=
 484
 485 # Known broken tests for the mips64 optimizing compiler backend.
 486 TEST_ART_BROKEN_OPTIMIZING_MIPS64_RUN_TESTS := \
 487
 488 ifeq (mips64,$(TARGET_ARCH))
 489   ifneq (,$(filter optimizing,$(COMPILER_TYPES)))
 490     ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \
 491         optimizing,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 492         $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \
 493         $(TEST_ART_BROKEN_OPTIMIZING_MIPS64_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 494   endif
 495 endif
 496
 497 TEST_ART_BROKEN_OPTIMIZING_MIPS64_RUN_TESTS :=
 498
 499 # Tests that should fail when the optimizing compiler compiles them non-debuggable.
 500 TEST_ART_BROKEN_OPTIMIZING_NONDEBUGGABLE_RUN_TESTS := \
 501   454-get-vreg \
 502   457-regs \
 503
 504 ifneq (,$(filter optimizing,$(COMPILER_TYPES)))
 505   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 506       optimizing,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 507       $(IMAGE_TYPES),$(PICTEST_TYPES),ndebuggable,$(TEST_ART_BROKEN_OPTIMIZING_NONDEBUGGABLE_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 508 endif
 509
 510 TEST_ART_BROKEN_OPTIMIZING_NONDEBUGGABLE_RUN_TESTS :=
 511
 512 # Tests that should fail when the optimizing compiler compiles them debuggable.
 513 TEST_ART_BROKEN_OPTIMIZING_DEBUGGABLE_RUN_TESTS := \
 514
 515 ifneq (,$(filter optimizing,$(COMPILER_TYPES)))
 516   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 517       optimizing,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 518       $(IMAGE_TYPES),$(PICTEST_TYPES),debuggable,$(TEST_ART_BROKEN_OPTIMIZING_DEBUGGABLE_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 519 endif
 520
 521 TEST_ART_BROKEN_OPTIMIZING_DEBUGGABLE_RUN_TESTS :=
 522
 523 # Tests that should fail in the read barrier configuration with the interpreter.
 524 TEST_ART_BROKEN_INTERPRETER_READ_BARRIER_RUN_TESTS :=
 525
 526 # Tests that should fail in the read barrier configuration with the Optimizing compiler (AOT).
 527 # 484: Baker's fast path based read barrier compiler instrumentation generates code containing
 528 #      more parallel moves on x86, thus some Checker assertions may fail.
 529 # 527: On ARM64, the read barrier instrumentation does not support the HArm64IntermediateAddress
 530 #      instruction yet (b/26601270).
 531 # 537: Expects an array copy to be intrinsified on x86-64, but calling-on-slowpath intrinsics are
 532 #      not yet handled in the read barrier configuration.
 533 TEST_ART_BROKEN_OPTIMIZING_READ_BARRIER_RUN_TESTS := \
 534   484-checker-register-hints \
 535   527-checker-array-access-split \
 536   537-checker-arraycopy
 537
 538 # Tests that should fail in the read barrier configuration with JIT (Optimizing compiler).
 539 TEST_ART_BROKEN_JIT_READ_BARRIER_RUN_TESTS :=
 540
 541 ifeq ($(ART_USE_READ_BARRIER),true)
 542   ifneq (,$(filter interpreter,$(COMPILER_TYPES)))
 543     ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \
 544         $(PREBUILD_TYPES),interpreter,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES), \
 545         $(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \
 546         $(TEST_ART_BROKEN_INTERPRETER_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 547   endif
 548
 549   ifneq (,$(filter optimizing,$(COMPILER_TYPES)))
 550     ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \
 551         $(PREBUILD_TYPES),optimizing,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES), \
 552         $(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \
 553         $(TEST_ART_BROKEN_OPTIMIZING_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 554   endif
 555
 556   ifneq (,$(filter jit,$(COMPILER_TYPES)))
 557     ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \
 558         $(PREBUILD_TYPES),jit,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES), \
 559         $(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \
 560         $(TEST_ART_BROKEN_JIT_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 561   endif
 562 endif
 563
 564 TEST_ART_BROKEN_OPTIMIZING_READ_BARRIER_RUN_TESTS :=
 565 TEST_ART_BROKEN_JIT_READ_BARRIER_RUN_TESTS :=
 566
 567 TEST_ART_BROKEN_NPIC_RUN_TESTS := 596-app-images
 568 ifneq (,$(filter npictest,$(PICTEST_TYPES)))
 569   ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \
 570       ${COMPILER_TYPES},$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 571       $(IMAGE_TYPES),npictest,$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_NPIC_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 572 endif
 573
 574 # Tests that should fail in the heap poisoning configuration with the Optimizing compiler.
 575 # 055: Exceeds run time limits due to heap poisoning instrumentation (on ARM and ARM64 devices).
 576 TEST_ART_BROKEN_OPTIMIZING_HEAP_POISONING_RUN_TESTS := \
 577   055-enum-performance
 578
 579 ifeq ($(ART_HEAP_POISONING),true)
 580   ifneq (,$(filter optimizing,$(COMPILER_TYPES)))
 581     ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \
 582         $(PREBUILD_TYPES),optimizing,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \
 583         $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \
 584         $(TEST_ART_BROKEN_OPTIMIZING_HEAP_POISONING_RUN_TESTS),$(ALL_ADDRESS_SIZES))
 585   endif
 586 endif
 587
 588 TEST_ART_BROKEN_OPTIMIZING_HEAP_POISONING_RUN_TESTS :=
 589
 590 # Clear variables ahead of appending to them when defining tests.
 591 $(foreach target, $(TARGET_TYPES), $(eval ART_RUN_TEST_$(call name-to-var,$(target))_RULES :=))
 592 $(foreach target, $(TARGET_TYPES), \
 593   $(foreach prebuild, $(PREBUILD_TYPES), \
 594     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(prebuild))_RULES :=)))
 595 $(foreach target, $(TARGET_TYPES), \
 596   $(foreach compiler, $(COMPILER_TYPES), \
 597     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(compiler))_RULES :=)))
 598 $(foreach target, $(TARGET_TYPES), \
 599   $(foreach relocate, $(RELOCATE_TYPES), \
 600     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(relocate))_RULES :=)))
 601 $(foreach target, $(TARGET_TYPES), \
 602   $(foreach trace, $(TRACE_TYPES), \
 603     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(trace))_RULES :=)))
 604 $(foreach target, $(TARGET_TYPES), \
 605   $(foreach gc, $(GC_TYPES), \
 606     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(gc))_RULES :=)))
 607 $(foreach target, $(TARGET_TYPES), \
 608   $(foreach jni, $(JNI_TYPES), \
 609     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(jni))_RULES :=)))
 610 $(foreach target, $(TARGET_TYPES), \
 611   $(foreach image, $(IMAGE_TYPES), \
 612     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(image))_RULES :=)))
 613 $(foreach target, $(TARGET_TYPES), \
 614   $(foreach test, $(TEST_ART_RUN_TESTS), \
 615     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(test))_RULES :=)))
 616 $(foreach target, $(TARGET_TYPES), \
 617   $(foreach address_size, $(ALL_ADDRESS_SIZES), \
 618     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(address_size))_RULES :=)))
 619 $(foreach target, $(TARGET_TYPES), \
 620   $(foreach run_type, $(RUN_TYPES), \
 621     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(run_type))_RULES :=)))
 622 $(foreach target, $(TARGET_TYPES), \
 623   $(foreach debuggable_type, $(DEBUGGABLE_TYPES), \
 624     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(debuggable_type))_RULES :=)))
 625
 626 # We need dex2oat and dalvikvm on the target as well as the core images (all images as we sync
 627 # only once).
 628 TEST_ART_TARGET_SYNC_DEPS += $(ART_TARGET_EXECUTABLES) $(TARGET_CORE_IMG_OUTS)
 629
 630 # Also need libarttest.
 631 TEST_ART_TARGET_SYNC_DEPS += $(ART_TARGET_TEST_OUT)/$(TARGET_ARCH)/libarttest.so
 632 TEST_ART_TARGET_SYNC_DEPS += $(ART_TARGET_TEST_OUT)/$(TARGET_ARCH)/libarttestd.so
 633 ifdef TARGET_2ND_ARCH
 634 TEST_ART_TARGET_SYNC_DEPS += $(ART_TARGET_TEST_OUT)/$(TARGET_2ND_ARCH)/libarttest.so
 635 TEST_ART_TARGET_SYNC_DEPS += $(ART_TARGET_TEST_OUT)/$(TARGET_2ND_ARCH)/libarttestd.so
 636 endif
 637
 638 # Also need libnativebridgetest.
 639 TEST_ART_TARGET_SYNC_DEPS += $(ART_TARGET_TEST_OUT)/$(TARGET_ARCH)/libnativebridgetest.so
 640 ifdef TARGET_2ND_ARCH
 641 TEST_ART_TARGET_SYNC_DEPS += $(ART_TARGET_TEST_OUT)/$(TARGET_2ND_ARCH)/libnativebridgetest.so
 642 endif
 643
 644 # All tests require the host executables. The tests also depend on the core images, but on
 645 # specific version depending on the compiler.
 646 ART_TEST_HOST_RUN_TEST_DEPENDENCIES := \
 647   $(ART_HOST_EXECUTABLES) \
 648   $(ART_HOST_OUT_SHARED_LIBRARIES)/libarttest$(ART_HOST_SHLIB_EXTENSION) \
 649   $(ART_HOST_OUT_SHARED_LIBRARIES)/libarttestd$(ART_HOST_SHLIB_EXTENSION) \
 650   $(ART_HOST_OUT_SHARED_LIBRARIES)/libnativebridgetest$(ART_HOST_SHLIB_EXTENSION) \
 651   $(ART_HOST_OUT_SHARED_LIBRARIES)/libjavacore$(ART_HOST_SHLIB_EXTENSION) \
 652   $(ART_HOST_OUT_SHARED_LIBRARIES)/libopenjdk$(ART_HOST_SHLIB_EXTENSION) \
 653   $(ART_HOST_OUT_SHARED_LIBRARIES)/libopenjdkd$(ART_HOST_SHLIB_EXTENSION)
 654
 655 ifneq ($(HOST_PREFER_32_BIT),true)
 656 ART_TEST_HOST_RUN_TEST_DEPENDENCIES += \
 657   $(2ND_ART_HOST_OUT_SHARED_LIBRARIES)/libarttest$(ART_HOST_SHLIB_EXTENSION) \
 658   $(2ND_ART_HOST_OUT_SHARED_LIBRARIES)/libarttestd$(ART_HOST_SHLIB_EXTENSION) \
 659   $(2ND_ART_HOST_OUT_SHARED_LIBRARIES)/libnativebridgetest$(ART_HOST_SHLIB_EXTENSION) \
 660   $(2ND_ART_HOST_OUT_SHARED_LIBRARIES)/libjavacore$(ART_HOST_SHLIB_EXTENSION) \
 661   $(2ND_ART_HOST_OUT_SHARED_LIBRARIES)/libopenjdk$(ART_HOST_SHLIB_EXTENSION) \
 662   $(2ND_ART_HOST_OUT_SHARED_LIBRARIES)/libopenjdkd$(ART_HOST_SHLIB_EXTENSION)
 663 endif
 664
 665 # Create a rule to build and run a tests following the form:
 666 # test-art-{1: host or target}-run-test-{2: debug ndebug}-{3: prebuild no-prebuild no-dex2oat}-
 667 #    {4: interpreter optimizing jit interp-ac}-
 668 #    {5: relocate nrelocate relocate-npatchoat}-
 669 #    {6: trace or ntrace}-{7: gcstress gcverify cms}-{8: forcecopy checkjni jni}-
 670 #    {9: no-image image picimage}-{10: pictest npictest}-
 671 #    {11: ndebuggable debuggable}-{12: test name}{13: 32 or 64}
 672 define define-test-art-run-test
 673   run_test_options :=
 674   prereq_rule :=
 675   test_groups :=
 676   uc_host_or_target :=
 677   jack_classpath :=
 678   ifeq ($(ART_TEST_WITH_STRACE),true)
 679     run_test_options += --strace
 680   endif
 681   ifeq ($(ART_TEST_RUN_TEST_ALWAYS_CLEAN),true)
 682     run_test_options += --always-clean
 683   endif
 684   ifeq ($(1),host)
 685     uc_host_or_target := HOST
 686     test_groups := ART_RUN_TEST_HOST_RULES
 687     run_test_options += --host
 688     prereq_rule := $(ART_TEST_HOST_RUN_TEST_DEPENDENCIES) $(HOST_JACK_CLASSPATH_DEPENDENCIES)
 689     jack_classpath := $(HOST_JACK_CLASSPATH)
 690   else
 691     ifeq ($(1),target)
 692       uc_host_or_target := TARGET
 693       test_groups := ART_RUN_TEST_TARGET_RULES
 694       prereq_rule := test-art-target-sync $(TARGET_JACK_CLASSPATH_DEPENDENCIES)
 695       jack_classpath := $(TARGET_JACK_CLASSPATH)
 696     else
 697       $$(error found $(1) expected $(TARGET_TYPES))
 698     endif
 699   endif
 700   ifeq ($(2),debug)
 701     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_DEBUG_RULES
 702   else
 703     ifeq ($(2),ndebug)
 704       test_groups += ART_RUN_TEST_$$(uc_host_or_target)_RELEASE_RULES
 705       run_test_options += -O
 706     else
 707       $$(error found $(2) expected $(RUN_TYPES))
 708     endif
 709   endif
 710   ifeq ($(3),prebuild)
 711     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_PREBUILD_RULES
 712     run_test_options += --prebuild
 713   else
 714     ifeq ($(3),no-prebuild)
 715       test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_PREBUILD_RULES
 716       run_test_options += --no-prebuild
 717     else
 718       ifeq ($(3),no-dex2oat)
 719         test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_DEX2OAT_RULES
 720         run_test_options += --no-prebuild --no-dex2oat
 721       else
 722         $$(error found $(3) expected $(PREBUILD_TYPES))
 723       endif
 724     endif
 725   endif
 726   ifeq ($(4),optimizing)
 727     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_OPTIMIZING_RULES
 728     run_test_options += --optimizing
 729   else
 730     ifeq ($(4),interpreter)
 731       test_groups += ART_RUN_TEST_$$(uc_host_or_target)_INTERPRETER_RULES
 732       run_test_options += --interpreter
 733     else ifeq ($(4),interp-ac)
 734       test_groups += ART_RUN_TEST_$$(uc_host_or_target)_INTERPRETER_ACCESS_CHECKS_RULES
 735       run_test_options += --interpreter --verify-soft-fail
 736     else
 737       ifeq ($(4),jit)
 738         test_groups += ART_RUN_TEST_$$(uc_host_or_target)_JIT_RULES
 739         run_test_options += --jit
 740       else
 741         $$(error found $(4) expected $(COMPILER_TYPES))
 742       endif
 743     endif
 744   endif
 745
 746   ifeq ($(5),relocate)
 747     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_RELOCATE_RULES
 748     run_test_options += --relocate
 749   else
 750     ifeq ($(5),no-relocate)
 751       test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_RELOCATE_RULES
 752       run_test_options += --no-relocate
 753     else
 754       ifeq ($(5),relocate-npatchoat)
 755         test_groups += ART_RUN_TEST_$$(uc_host_or_target)_RELOCATE_NO_PATCHOAT_RULES
 756         run_test_options += --relocate --no-patchoat
 757       else
 758         $$(error found $(5) expected $(RELOCATE_TYPES))
 759       endif
 760     endif
 761   endif
 762   ifeq ($(6),trace)
 763     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_TRACE_RULES
 764     run_test_options += --trace
 765   else
 766     ifeq ($(6),ntrace)
 767       test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_TRACE_RULES
 768     else
 769       ifeq ($(6),stream)
 770         # Group streaming under normal tracing rules.
 771         test_groups += ART_RUN_TEST_$$(uc_host_or_target)_TRACE_RULES
 772         run_test_options += --trace --stream
 773       else
 774         $$(error found $(6) expected $(TRACE_TYPES))
 775       endif
 776     endif
 777   endif
 778   ifeq ($(7),gcverify)
 779     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_GCVERIFY_RULES
 780     run_test_options += --gcverify
 781   else
 782     ifeq ($(7),gcstress)
 783       test_groups += ART_RUN_TEST_$$(uc_host_or_target)_GCSTRESS_RULES
 784       run_test_options += --gcstress
 785     else
 786       ifeq ($(7),cms)
 787         test_groups += ART_RUN_TEST_$$(uc_host_or_target)_CMS_RULES
 788       else
 789         $$(error found $(7) expected $(GC_TYPES))
 790       endif
 791     endif
 792   endif
 793   ifeq ($(8),forcecopy)
 794     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_FORCECOPY_RULES
 795     run_test_options += --runtime-option -Xjniopts:forcecopy
 796     ifneq ($$(ART_TEST_JNI_FORCECOPY),true)
 797       skip_test := true
 798     endif
 799   else
 800     ifeq ($(8),checkjni)
 801       test_groups += ART_RUN_TEST_$$(uc_host_or_target)_CHECKJNI_RULES
 802       run_test_options += --runtime-option -Xcheck:jni
 803     else
 804       ifeq ($(8),jni)
 805         test_groups += ART_RUN_TEST_$$(uc_host_or_target)_JNI_RULES
 806       else
 807         $$(error found $(8) expected $(JNI_TYPES))
 808       endif
 809     endif
 810   endif
 811   image_suffix := $(4)
 812   ifeq ($(9),no-image)
 813     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_IMAGE_RULES
 814     run_test_options += --no-image
 815     # Add the core dependency. This is required for pre-building.
 816     ifeq ($(1),host)
 817       prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_no-pic_$(13))
 818     else
 819       prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_no-pic_$(13))
 820     endif
 821   else
 822     ifeq ($(9),image)
 823       test_groups += ART_RUN_TEST_$$(uc_host_or_target)_IMAGE_RULES
 824       # Add the core dependency.
 825       ifeq ($(1),host)
 826         prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_no-pic_$(13))
 827       else
 828         prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_no-pic_$(13))
 829       endif
 830     else
 831       ifeq ($(9),picimage)
 832         test_groups += ART_RUN_TEST_$$(uc_host_or_target)_PICIMAGE_RULES
 833         run_test_options += --pic-image
 834         ifeq ($(1),host)
 835           prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_pic_$(13))
 836         else
 837           prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_pic_$(13))
 838         endif
 839       else
 840         ifeq ($(9),multiimage)
 841           test_groups += ART_RUN_TEST_$$(uc_host_or_target)_IMAGE_RULES
 842           run_test_options += --multi-image
 843                 ifeq ($(1),host)
 844                         prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_no-pic_multi_$(13))
 845                 else
 846                         prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_no-pic_multi_$(13))
 847                 endif
 848         else
 849           ifeq ($(9),multipicimage)
 850             test_groups += ART_RUN_TEST_$$(uc_host_or_target)_PICIMAGE_RULES
 851                         run_test_options += --pic-image --multi-image
 852                         ifeq ($(1),host)
 853                         prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_pic_multi_$(13))
 854                         else
 855                         prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_pic_multi_$(13))
 856                         endif
 857           else
 858             $$(error found $(9) expected $(IMAGE_TYPES))
 859           endif
 860         endif
 861       endif
 862     endif
 863   endif
 864   ifeq ($(10),pictest)
 865     run_test_options += --pic-test
 866   else
 867     ifeq ($(10),npictest)
 868       # Nothing to be done.
 869     else
 870       $$(error found $(10) expected $(PICTEST_TYPES))
 871     endif
 872   endif
 873   ifeq ($(11),debuggable)
 874     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_DEBUGGABLE_RULES
 875     run_test_options += --debuggable
 876   else
 877     ifeq ($(11),ndebuggable)
 878     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NONDEBUGGABLE_RULES
 879       # Nothing to be done.
 880     else
 881       $$(error found $(11) expected $(DEBUGGABLE_TYPES))
 882     endif
 883   endif
 884   # $(12) is the test name.
 885   test_groups += ART_RUN_TEST_$$(uc_host_or_target)_$(call name-to-var,$(12))_RULES
 886   ifeq ($(13),64)
 887     test_groups += ART_RUN_TEST_$$(uc_host_or_target)_64_RULES
 888     run_test_options += --64
 889   else
 890     ifeq ($(13),32)
 891       test_groups += ART_RUN_TEST_$$(uc_host_or_target)_32_RULES
 892     else
 893       $$(error found $(13) expected $(ALL_ADDRESS_SIZES))
 894     endif
 895   endif
 896   # Override of host instruction-set-features. Required to test advanced x86 intrinsics. The
 897   # conditionals aren't really correct, they will fail to do the right thing on a 32-bit only
 898   # host. However, this isn't common enough to worry here and make the conditions complicated.
 899   ifneq ($(DEX2OAT_HOST_INSTRUCTION_SET_FEATURES),)
 900     ifeq ($(13),64)
 901       run_test_options += --instruction-set-features $(DEX2OAT_HOST_INSTRUCTION_SET_FEATURES)
 902     endif
 903   endif
 904   ifneq ($($(HOST_2ND_ARCH_VAR_PREFIX)DEX2OAT_HOST_INSTRUCTION_SET_FEATURES),)
 905     ifeq ($(13),32)
 906       run_test_options += --instruction-set-features $($(HOST_2ND_ARCH_VAR_PREFIX)DEX2OAT_HOST_INSTRUCTION_SET_FEATURES)
 907     endif
 908   endif
 909   run_test_rule_name := test-art-$(1)-run-test-$(2)-$(3)-$(4)-$(5)-$(6)-$(7)-$(8)-$(9)-$(10)-$(11)-$(12)$(13)
 910   run_test_options := --output-path $(ART_HOST_TEST_DIR)/run-test-output/$$(run_test_rule_name) \
 911       $$(run_test_options)
 912   ifneq ($(ART_TEST_ANDROID_ROOT),)
 913     run_test_options := --android-root $(ART_TEST_ANDROID_ROOT) $$(run_test_options)
 914   endif
 915   ifeq ($(ART_TEST_QUIET),true)
 916     run_test_options += --quiet
 917   endif
 918 $$(run_test_rule_name): PRIVATE_RUN_TEST_OPTIONS := $$(run_test_options)
 919 $$(run_test_rule_name): PRIVATE_JACK_CLASSPATH := $$(jack_classpath)
 920 .PHONY: $$(run_test_rule_name)
 921 $$(run_test_rule_name): $(TEST_ART_RUN_TEST_DEPENDENCIES) $(HOST_OUT_EXECUTABLES)/hprof-conv $$(prereq_rule) | $(TEST_ART_RUN_TEST_ORDERONLY_DEPENDENCIES)
 922         $(hide) $$(call ART_TEST_SKIP,$$@) && \
 923           DX=$(abspath $(DX)) \
 924             JASMIN=$(abspath $(HOST_OUT_EXECUTABLES)/jasmin) \
 925             SMALI=$(abspath $(HOST_OUT_EXECUTABLES)/smali) \
 926             DXMERGER=$(abspath $(HOST_OUT_EXECUTABLES)/dexmerger) \
 927             JACK_VERSION=$(JACK_DEFAULT_VERSION) \
 928             JACK=$(abspath $(JACK)) \
 929             JACK_VERSION=$(JACK_DEFAULT_VERSION) \
 930             JACK_CLASSPATH=$$(PRIVATE_JACK_CLASSPATH) \
 931             art/test/run-test $$(PRIVATE_RUN_TEST_OPTIONS) $(12) \
 932               && $$(call ART_TEST_PASSED,$$@) || $$(call ART_TEST_FAILED,$$@)
 933         $$(hide) (echo $(MAKECMDGOALS) | grep -q $$@ && \
 934           echo "run-test run as top-level target, removing test directory $(ART_HOST_TEST_DIR)" && \
 935           rm -r $(ART_HOST_TEST_DIR)) || true
 936
 937   $$(foreach test_group,$$(test_groups), $$(eval $$(value test_group) += $$(run_test_rule_name)))
 938
 939   # Clear locally defined variables.
 940   uc_host_or_target :=
 941   test_groups :=
 942   run_test_options :=
 943   run_test_rule_name :=
 944   prereq_rule :=
 945   jack_classpath :=
 946 endef  # define-test-art-run-test
 947
 948 $(foreach target, $(TARGET_TYPES), \
 949   $(foreach test, $(TEST_ART_RUN_TESTS), \
 950     $(foreach run_type, $(RUN_TYPES), \
 951       $(foreach address_size, $(ADDRESS_SIZES_$(call name-to-var,$(target))), \
 952         $(foreach prebuild, $(PREBUILD_TYPES), \
 953           $(foreach compiler, $(COMPILER_TYPES), \
 954             $(foreach relocate, $(RELOCATE_TYPES), \
 955               $(foreach trace, $(TRACE_TYPES), \
 956                 $(foreach gc, $(GC_TYPES), \
 957                   $(foreach jni, $(JNI_TYPES), \
 958                     $(foreach image, $(IMAGE_TYPES), \
 959                       $(foreach pictest, $(PICTEST_TYPES), \
 960                         $(foreach debuggable, $(DEBUGGABLE_TYPES), \
 961                           $(eval $(call define-test-art-run-test,$(target),$(run_type),$(prebuild),$(compiler),$(relocate),$(trace),$(gc),$(jni),$(image),$(pictest),$(debuggable),$(test),$(address_size))) \
 962                   )))))))))))))
 963 define-test-art-run-test :=
 964
 965 # Define a phony rule whose purpose is to test its prerequisites.
 966 # $(1): host or target
 967 # $(2): list of prerequisites
 968 define define-test-art-run-test-group
 969 .PHONY: $(1)
 970 $(1): $(2)
 971         $(hide) $$(call ART_TEST_PREREQ_FINISHED,$$@)
 972
 973 endef  # define-test-art-run-test-group
 974
 975
 976 $(foreach target, $(TARGET_TYPES), $(eval \
 977   $(call define-test-art-run-test-group,test-art-$(target)-run-test,$(ART_RUN_TEST_$(call name-to-var,$(target))_RULES))))
 978 $(foreach target, $(TARGET_TYPES), \
 979   $(foreach prebuild, $(PREBUILD_TYPES), $(eval \
 980     $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(prebuild),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(prebuild))_RULES)))))
 981 $(foreach target, $(TARGET_TYPES), \
 982   $(foreach run-type, $(RUN_TYPES), $(eval \
 983     $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(run-type),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(run-type))_RULES)))))
 984 $(foreach target, $(TARGET_TYPES), \
 985   $(foreach compiler, $(COMPILER_TYPES), $(eval \
 986     $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(compiler),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(compiler))_RULES)))))
 987 $(foreach target, $(TARGET_TYPES), \
 988   $(foreach relocate, $(RELOCATE_TYPES), $(eval \
 989     $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(relocate),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(relocate))_RULES)))))
 990 $(foreach target, $(TARGET_TYPES), \
 991   $(foreach trace, $(TRACE_TYPES), $(eval \
 992     $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(trace),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(trace))_RULES)))))
 993 $(foreach target, $(TARGET_TYPES), \
 994   $(foreach gc, $(GC_TYPES), $(eval \
 995     $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(gc),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(gc))_RULES)))))
 996 $(foreach target, $(TARGET_TYPES), \
 997   $(foreach jni, $(JNI_TYPES), $(eval \
 998     $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(jni),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(jni))_RULES)))))
 999 $(foreach target, $(TARGET_TYPES), \
1000   $(foreach debuggable, $(DEBUGGABLE_TYPES), $(eval \
1001     $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(debuggable),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(debuggable))_RULES)))))
1002 $(foreach target, $(TARGET_TYPES), \
1003   $(foreach image, $(IMAGE_TYPES), $(eval \
1004     $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(image),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(image))_RULES)))))
1005 $(foreach target, $(TARGET_TYPES), \
1006   $(foreach test, $(TEST_ART_RUN_TESTS), $(eval \
1007     $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(test),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(test))_RULES)))))
1008 $(foreach target, $(TARGET_TYPES), \
1009   $(foreach address_size, $(ADDRESS_SIZES_$(call name-to-var,$(target))), $(eval \
1010     $(call define-test-art-run-test-group,test-art-$(target)-run-test$(address_size),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(address_size)_RULES)))))
1011
1012 # Clear variables now we're finished with them.
1013 $(foreach target, $(TARGET_TYPES), $(eval ART_RUN_TEST_$(call name-to-var,$(target))_RULES :=))
1014 $(foreach target, $(TARGET_TYPES), \
1015   $(foreach prebuild, $(PREBUILD_TYPES), \
1016     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(prebuild))_RULES :=)))
1017 $(foreach target, $(TARGET_TYPES), \
1018   $(foreach compiler, $(COMPILER_TYPES), \
1019     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(compiler))_RULES :=)))
1020 $(foreach target, $(TARGET_TYPES), \
1021   $(foreach relocate, $(RELOCATE_TYPES), \
1022     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(relocate))_RULES :=)))
1023 $(foreach target, $(TARGET_TYPES), \
1024   $(foreach trace, $(TRACE_TYPES), \
1025     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(trace))_RULES :=)))
1026 $(foreach target, $(TARGET_TYPES), \
1027   $(foreach gc, $(GC_TYPES), \
1028     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(gc))_RULES :=)))
1029 $(foreach target, $(TARGET_TYPES), \
1030   $(foreach jni, $(JNI_TYPES), \
1031     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(jni))_RULES :=)))
1032 $(foreach target, $(TARGET_TYPES), \
1033   $(foreach debuggable, $(DEBUGGABLE_TYPES), \
1034     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(debuggable))_RULES :=)))
1035 $(foreach target, $(TARGET_TYPES), \
1036   $(foreach image, $(IMAGE_TYPES), \
1037     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(image))_RULES :=)))
1038 $(foreach target, $(TARGET_TYPES), \
1039   $(foreach test, $(TEST_ART_RUN_TESTS), \
1040     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(test))_RULES :=)))
1041 $(foreach target, $(TARGET_TYPES), \
1042   $(foreach address_size, $(ALL_ADDRESS_SIZES), \
1043     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(address_size))_RULES :=)))
1044 $(foreach target, $(TARGET_TYPES), \
1045   $(foreach run_type, $(RUN_TYPES), \
1046     $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(run_type))_RULES :=)))
1047 define-test-art-run-test-group :=
1048 TARGET_TYPES :=
1049 PREBUILD_TYPES :=
1050 COMPILER_TYPES :=
1051 RELOCATE_TYPES :=
1052 TRACE_TYPES :=
1053 GC_TYPES :=
1054 JNI_TYPES :=
1055 IMAGE_TYPES :=
1056 ADDRESS_SIZES_TARGET :=
1057 ADDRESS_SIZES_HOST :=
1058 ALL_ADDRESS_SIZES :=
1059 RUN_TYPES :=
1060 DEBUGGABLE_TYPES :=
1061
1062 include $(LOCAL_PATH)/Android.libarttest.mk
1063 include art/test/Android.libnativebridgetest.mk