diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt index 1c433f10e1..d76f7384c2 100644 --- a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt @@ -435,71 +435,70 @@ private fun replayClassCandidates( ) } val scenarios = source.record.scenarios.withIndex().associate { it.value.id to it.index } - for (index in search.search.candidates.indices.drop(events.size)) { - source.verify() - val candidate = search.search.candidates[index] - val scenarioId = requireNotNull(candidate.entryPointId) - val scenarioIndex = scenarios.getValue(scenarioId) - val function = source.record.functions[scenarioIndex] - val observations = source.record.observations[scenarioIndex] - val replayStarted = TimeSource.Monotonic.markNow() - val original = runCatching { - OriginalTypeScriptClassReplayer( - sourceRoot = source.harness.sourceRoot, - function = function, - observationFields = observations, - ).use { it.replay(inputs = candidate.inputs, timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS) } - } - val instrumentation = source.record.instrumentation - val instrumented = runCatching { - OriginalTypeScriptClassReplayer( - sourceRoot = source.harness.sourceRoot, - function = function, - observationFields = observations, - sourceOverride = instrumentation.instrumentedSource, - overrideSourceFile = source.record.classCase.surface.sourceFile, - counterGlobalName = instrumentation.hitGlobalName, - ).use { it.replay(inputs = candidate.inputs, timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS) } - } - source.verify() - val originalValue = original.getOrNull() - val instrumentedValue = instrumented.getOrNull() - val validation = runCatching { - require(originalValue != null && instrumentedValue != null) { "Concrete replay failed" } - require(originalValue.inputs == candidate.inputs && instrumentedValue.inputs == candidate.inputs) { - "Class replay input snapshot differs from candidate" + CallsClassScenarioReplayers(source.harness, source.record.instrumentation).use { replayers -> + for (index in search.search.candidates.indices.drop(events.size)) { + source.verify() + val candidate = search.search.candidates[index] + val scenarioId = requireNotNull(candidate.entryPointId) + val scenarioIndex = scenarios.getValue(scenarioId) + val replayStarted = TimeSource.Monotonic.markNow() + val original = runCatching { + replayers.replay( + scenarioIndex = scenarioIndex, + inputs = candidate.inputs, + timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS, + instrumented = false, + ) } - instrumentation.universe.checkedCoveredIds(instrumentedValue.coveredStatementKeys) - classCandidateStrictOracle(candidate, originalValue, instrumentedValue) - } - val failures = listOfNotNull( - original.exceptionOrNull()?.let { "original: ${it.message ?: it::class.java.name}" }, - instrumented.exceptionOrNull()?.let { "instrumented: ${it.message ?: it::class.java.name}" }, - validation.exceptionOrNull()?.let { "validation: ${it.message ?: it::class.java.name}" }, - ) - val replayError = failures.takeIf { it.isNotEmpty() }?.joinToString(separator = "; ") - val event = CallsClassReplayEvent( - index = index, - scenarioId = scenarioId, - original = originalValue, - instrumented = instrumentedValue, - oracle = validation.getOrNull() ?: CallsClassOracleResult( - accepted = false, - diagnostic = "Concrete replay failed", - ), - error = replayError, - replayElapsedMillis = replayStarted.elapsedNow().inWholeMilliseconds, - nativeCoverage = callsClassNativeCoverageAcceptance( + val instrumentation = source.record.instrumentation + val instrumented = runCatching { + replayers.replay( + scenarioIndex = scenarioIndex, + inputs = candidate.inputs, + timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS, + instrumented = true, + ) + } + source.verify() + val originalValue = original.getOrNull() + val instrumentedValue = instrumented.getOrNull() + val validation = runCatching { + require(originalValue != null && instrumentedValue != null) { "Concrete replay failed" } + require(originalValue.inputs == candidate.inputs && instrumentedValue.inputs == candidate.inputs) { + "Class replay input snapshot differs from candidate" + } + instrumentation.universe.checkedCoveredIds(instrumentedValue.coveredStatementKeys) + classCandidateStrictOracle(candidate, originalValue, instrumentedValue) + } + val failures = listOfNotNull( + original.exceptionOrNull()?.let { "original: ${it.message ?: it::class.java.name}" }, + instrumented.exceptionOrNull()?.let { "instrumented: ${it.message ?: it::class.java.name}" }, + validation.exceptionOrNull()?.let { "validation: ${it.message ?: it::class.java.name}" }, + ) + val replayError = failures.takeIf { it.isNotEmpty() }?.joinToString(separator = "; ") + val event = CallsClassReplayEvent( + index = index, + scenarioId = scenarioId, original = originalValue, instrumented = instrumentedValue, - universe = instrumentation.universe, + oracle = validation.getOrNull() ?: CallsClassOracleResult( + accepted = false, + diagnostic = "Concrete replay failed", + ), error = replayError, - ), - origin = candidate.origin, - strictOracleApplicable = candidate.origin == CallsCoverageCandidateOrigin.TERMINATED, - ) - appendCallsJournalDurably(path, encodeCallsArtifact(event)) - events += event + replayElapsedMillis = replayStarted.elapsedNow().inWholeMilliseconds, + nativeCoverage = callsClassNativeCoverageAcceptance( + original = originalValue, + instrumented = instrumentedValue, + universe = instrumentation.universe, + error = replayError, + ), + origin = candidate.origin, + strictOracleApplicable = candidate.origin == CallsCoverageCandidateOrigin.TERMINATED, + ) + appendCallsJournalDurably(path, encodeCallsArtifact(event)) + events += event + } } return events diff --git a/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassScenarioReplayers.kt b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassScenarioReplayers.kt new file mode 100644 index 0000000000..186da397a3 --- /dev/null +++ b/usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassScenarioReplayers.kt @@ -0,0 +1,51 @@ +package org.usvm.ts.calls + +import org.usvm.ts.pbt.model.JsConcreteValue + +/** Reuses only source preparation; every replay still starts a fresh supervised Node.js process. */ +internal class CallsClassScenarioReplayers( + private val harness: CallsClassPreparedHarness, + private val instrumentation: CallsClassSourceInstrumentation, +) : AutoCloseable { + private val originalReplayers = mutableMapOf() + private val instrumentedReplayers = mutableMapOf() + + fun replay( + scenarioIndex: Int, + inputs: List, + timeoutMillis: Long, + instrumented: Boolean, + ): CallsClassReplayObservation { + val replayers = if (instrumented) instrumentedReplayers else originalReplayers + val replayer = replayers.getOrPut(scenarioIndex) { + OriginalTypeScriptClassReplayer( + sourceRoot = harness.sourceRoot, + function = harness.functions[scenarioIndex], + observationFields = harness.observations[scenarioIndex], + sourceOverride = if (instrumented) instrumentation.instrumentedSource else null, + overrideSourceFile = if (instrumented) harness.surface.sourceFile else null, + counterGlobalName = if (instrumented) instrumentation.hitGlobalName else null, + ) + } + + return replayer.replay(inputs = inputs, timeoutMillis = timeoutMillis) + } + + override fun close() { + var failure: Throwable? = null + for (replayer in originalReplayers.values + instrumentedReplayers.values) { + val error = runCatching { replayer.close() }.exceptionOrNull() + if (error == null) continue + + val firstFailure = failure + if (firstFailure == null) { + failure = error + } else { + firstFailure.addSuppressed(error) + } + } + originalReplayers.clear() + instrumentedReplayers.clear() + failure?.let { throw it } + } +} diff --git a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt index 11abffc97c..c42af80f36 100644 --- a/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt +++ b/usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt @@ -2,6 +2,7 @@ package org.usvm.ts.calls import org.junit.jupiter.api.io.TempDir import org.usvm.SolverType +import org.usvm.ts.pbt.fastcheck.PbtBackendException import org.usvm.ts.pbt.model.ArrayDomain import org.usvm.ts.pbt.model.ConstantDomain import org.usvm.ts.pbt.model.IntegerDomain @@ -62,6 +63,103 @@ class CallsClassHarnessOracleTest { } } + @Test + fun `cached scenario preparation keeps fresh module state inputs and counters for every replay`() { + val statefulSource = "let replayRuns = 0;\n" + source.replace( + "this.count = start;", + "replayRuns++; this.count = start + replayRuns; if (start === 0) this.count += 10;", + ) + Files.writeString(directory.resolve("Counter.ts"), statefulSource) + val harness = prepareCallsClassHarness(directory, project(), surface()) + val instrumentation = CallsClassSourceInstrumentation.create( + sourceRoot = directory, + sourceFile = "Counter.ts", + className = "Counter", + ) + + val observations = CallsClassScenarioReplayers(harness, instrumentation).use { replayers -> + listOf(2, 0, 2).map { start -> + val inputs = listOf(number(start)) + val original = replayers.replay( + scenarioIndex = 0, + inputs = inputs, + timeoutMillis = 20_000, + instrumented = false, + ) + val instrumented = replayers.replay( + scenarioIndex = 0, + inputs = inputs, + timeoutMillis = 20_000, + instrumented = true, + ) + + assertEquals(inputs, original.inputs) + assertEquals(inputs, instrumented.inputs) + val nativeCoverage = callsClassNativeCoverageAcceptance( + original, + instrumented, + instrumentation.universe, + ) + assertTrue(nativeCoverage.accepted) + original to instrumented + } + } + + val initialStates = observations.map { it.first.observation.values.getValue("state_0_count") } + assertEquals(listOf(number(3), number(11), number(3)), initialStates) + assertEquals(observations.first().second.coveredStatementKeys, observations.last().second.coveredStatementKeys) + val branchOnlyHits = observations[1].second.coveredStatementKeys - observations[0].second.coveredStatementKeys + assertTrue(branchOnlyHits.isNotEmpty()) + } + + @Test + fun `cached replay timeout never returns a preceding observation and next input still succeeds`() { + val sourceWithLoop = source.replace( + "this.count = start;", + "if (start === 1) while (true) {} this.count = start;", + ) + Files.writeString(directory.resolve("Counter.ts"), sourceWithLoop) + val harness = prepareCallsClassHarness(directory, project(), surface()) + val instrumentation = CallsClassSourceInstrumentation.create( + sourceRoot = directory, + sourceFile = "Counter.ts", + className = "Counter", + ) + + CallsClassScenarioReplayers(harness, instrumentation).use { replayers -> + for (instrumented in listOf(false, true)) { + val first = replayers.replay( + scenarioIndex = 0, + inputs = listOf(number(2)), + timeoutMillis = 20_000, + instrumented = instrumented, + ) + + val timeout = assertFailsWith { + replayers.replay( + scenarioIndex = 0, + inputs = listOf(number(1)), + timeoutMillis = 50, + instrumented = instrumented, + ) + } + assertEquals("backend.process.timeout", timeout.code) + + val next = replayers.replay( + scenarioIndex = 0, + inputs = listOf(number(3)), + timeoutMillis = 20_000, + instrumented = instrumented, + ) + + assertEquals(listOf(number(2)), first.inputs) + assertEquals(number(2), first.observation.values.getValue("state_0_count")) + assertEquals(listOf(number(3)), next.inputs) + assertEquals(number(3), next.observation.values.getValue("state_0_count")) + } + } + } + @Test fun `object parameter transport stays frozen when callable mutates it`() { val harness = fixture()