diff --git a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt index 8303a67aa6..0bcfd29bd7 100644 --- a/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt +++ b/utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt @@ -597,6 +597,21 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS var disableUnsatChecking by getBooleanProperty(false) // endregion + + // region Spring-related options + + /** + * When generating integration tests we only partially reset context in between executions to save time. + * For example, entity id generators do not get reset. It may lead to non-reproduceable results if + * IDs leak to the output of the method under test. + * + * To cope with that, we rerun executions that are left after minimization, fully resetting Spring context + * between executions. However, full context reset is slow, so we use this setting to limit number of + * tests per method that are rerun with full context reset in case minimization outputs too many tests. + */ + var maxSpringContextResetsPerMethod by getIntProperty(25, 0, Int.MAX_VALUE) + + // endregion } /** diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt index a4b33910e3..55feae324f 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/context/custom/RerunningConcreteExecutionContext.kt @@ -13,6 +13,7 @@ import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrument class RerunningConcreteExecutionContext( private val delegateContext: ConcreteExecutionContext, + private val maxRerunsPerMethod: Int, private val rerunTimeoutInMillis: Long = 10L * UtSettings.concreteExecutionDefaultTimeoutInInstrumentedProcessMillis, ) : ConcreteExecutionContext by delegateContext { companion object { @@ -23,31 +24,40 @@ class RerunningConcreteExecutionContext( executions: List, methodUnderTest: ExecutableId, rerunExecutor: ConcreteExecutor, - ): List = delegateContext.transformExecutionsAfterMinimization( - executions, - methodUnderTest, - rerunExecutor - ).map { execution -> - runBlocking { - val result = try { - rerunExecutor.executeConcretely( - methodUnderTest = methodUnderTest, - stateBefore = execution.stateBefore, - instrumentation = emptyList(), - timeoutInMillis = rerunTimeoutInMillis, - isRerun = true, - ) - } catch (e: Throwable) { - // we can't update execution result if we don't have a result - logger.warn(e) { "Rerun failed, keeping original result for execution [$execution]" } - return@runBlocking execution - } - execution.copy( - stateBefore = result.stateBefore, - stateAfter = result.stateAfter, - result = result.result, - coverage = result.coverage, - ) - } + ): List { + @Suppress("NAME_SHADOWING") + val executions = delegateContext.transformExecutionsAfterMinimization( + executions, + methodUnderTest, + rerunExecutor + ) + // it's better to rerun executions with non-empty coverage, + // because executions with empty coverage are often duplicated + .sortedBy { it.coverage?.coveredInstructions.isNullOrEmpty() } + return executions + .take(maxRerunsPerMethod) + .map { execution -> + runBlocking { + val result = try { + rerunExecutor.executeConcretely( + methodUnderTest = methodUnderTest, + stateBefore = execution.stateBefore, + instrumentation = emptyList(), + timeoutInMillis = rerunTimeoutInMillis, + isRerun = true, + ) + } catch (e: Throwable) { + // we can't update execution result if we don't have a result + logger.warn(e) { "Rerun failed, keeping original result for execution [$execution]" } + return@runBlocking execution + } + execution.copy( + stateBefore = result.stateBefore, + stateAfter = result.stateAfter, + result = result.result, + coverage = result.coverage, + ) + } + } + executions.drop(maxRerunsPerMethod) } } \ No newline at end of file diff --git a/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt b/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt index bdf5745199..f70bc85ae3 100644 --- a/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt +++ b/utbot-spring-framework/src/main/kotlin/org/utbot/framework/context/spring/SpringApplicationContextImpl.kt @@ -5,6 +5,7 @@ import org.utbot.common.dynamicPropertiesOf import org.utbot.common.isAbstract import org.utbot.common.isStatic import org.utbot.common.withValue +import org.utbot.framework.UtSettings import org.utbot.framework.codegen.generator.AbstractCodeGenerator import org.utbot.framework.codegen.generator.CodeGeneratorParams import org.utbot.framework.codegen.generator.SpringCodeGenerator @@ -118,7 +119,8 @@ class SpringApplicationContextImpl( delegateConcreteExecutionContext, classpathWithoutDependencies, springApplicationContext = this - ) + ), + maxRerunsPerMethod = UtSettings.maxSpringContextResetsPerMethod ) }.transformInstrumentationFactory { delegateInstrumentationFactory -> RemovingConstructFailsUtExecutionInstrumentation.Factory(delegateInstrumentationFactory)