Skip to content

Reuse source preparation during class candidate replay #521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
CaelmBleidd merged 1 commit into from
Oct 15, 2026
+209 −61
Merged

Reuse source preparation during class candidate replay #521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 60 additions & 61 deletions usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassExperimentCli.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -435,71 +435,70 @@ private fun replayClassCandidates(
)
}
val scenarios = source.record.scenarios.withIndex().associate { it.value.id to it.index }
for (index in search.search.candidates.indices.drop(events.size)) {
source.verify()
val candidate = search.search.candidates[index]
val scenarioId = requireNotNull(candidate.entryPointId)
val scenarioIndex = scenarios.getValue(scenarioId)
val function = source.record.functions[scenarioIndex]
val observations = source.record.observations[scenarioIndex]
val replayStarted = TimeSource.Monotonic.markNow()
val original = runCatching {
OriginalTypeScriptClassReplayer(
sourceRoot = source.harness.sourceRoot,
function = function,
observationFields = observations,
).use { it.replay(inputs = candidate.inputs, timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS) }
}
val instrumentation = source.record.instrumentation
val instrumented = runCatching {
OriginalTypeScriptClassReplayer(
sourceRoot = source.harness.sourceRoot,
function = function,
observationFields = observations,
sourceOverride = instrumentation.instrumentedSource,
overrideSourceFile = source.record.classCase.surface.sourceFile,
counterGlobalName = instrumentation.hitGlobalName,
).use { it.replay(inputs = candidate.inputs, timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS) }
}
source.verify()
val originalValue = original.getOrNull()
val instrumentedValue = instrumented.getOrNull()
val validation = runCatching {
require(originalValue != null && instrumentedValue != null) { "Concrete replay failed" }
require(originalValue.inputs == candidate.inputs && instrumentedValue.inputs == candidate.inputs) {
"Class replay input snapshot differs from candidate"
CallsClassScenarioReplayers(source.harness, source.record.instrumentation).use { replayers ->
for (index in search.search.candidates.indices.drop(events.size)) {
source.verify()
val candidate = search.search.candidates[index]
val scenarioId = requireNotNull(candidate.entryPointId)
val scenarioIndex = scenarios.getValue(scenarioId)
val replayStarted = TimeSource.Monotonic.markNow()
val original = runCatching {
replayers.replay(
scenarioIndex = scenarioIndex,
inputs = candidate.inputs,
timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS,
instrumented = false,
)
}
instrumentation.universe.checkedCoveredIds(instrumentedValue.coveredStatementKeys)
classCandidateStrictOracle(candidate, originalValue, instrumentedValue)
}
val failures = listOfNotNull(
original.exceptionOrNull()?.let { "original: ${it.message ?: it::class.java.name}" },
instrumented.exceptionOrNull()?.let { "instrumented: ${it.message ?: it::class.java.name}" },
validation.exceptionOrNull()?.let { "validation: ${it.message ?: it::class.java.name}" },
)
val replayError = failures.takeIf { it.isNotEmpty() }?.joinToString(separator = "; ")
val event = CallsClassReplayEvent(
index = index,
scenarioId = scenarioId,
original = originalValue,
instrumented = instrumentedValue,
oracle = validation.getOrNull() ?: CallsClassOracleResult(
accepted = false,
diagnostic = "Concrete replay failed",
),
error = replayError,
replayElapsedMillis = replayStarted.elapsedNow().inWholeMilliseconds,
nativeCoverage = callsClassNativeCoverageAcceptance(
val instrumentation = source.record.instrumentation
val instrumented = runCatching {
replayers.replay(
scenarioIndex = scenarioIndex,
inputs = candidate.inputs,
timeoutMillis = CLASS_REPLAY_TIMEOUT_MILLIS,
instrumented = true,
)
}
source.verify()
val originalValue = original.getOrNull()
val instrumentedValue = instrumented.getOrNull()
val validation = runCatching {
require(originalValue != null && instrumentedValue != null) { "Concrete replay failed" }
require(originalValue.inputs == candidate.inputs && instrumentedValue.inputs == candidate.inputs) {
"Class replay input snapshot differs from candidate"
}
instrumentation.universe.checkedCoveredIds(instrumentedValue.coveredStatementKeys)
classCandidateStrictOracle(candidate, originalValue, instrumentedValue)
}
val failures = listOfNotNull(
original.exceptionOrNull()?.let { "original: ${it.message ?: it::class.java.name}" },
instrumented.exceptionOrNull()?.let { "instrumented: ${it.message ?: it::class.java.name}" },
validation.exceptionOrNull()?.let { "validation: ${it.message ?: it::class.java.name}" },
)
val replayError = failures.takeIf { it.isNotEmpty() }?.joinToString(separator = "; ")
val event = CallsClassReplayEvent(
index = index,
scenarioId = scenarioId,
original = originalValue,
instrumented = instrumentedValue,
universe = instrumentation.universe,
oracle = validation.getOrNull() ?: CallsClassOracleResult(
accepted = false,
diagnostic = "Concrete replay failed",
),
error = replayError,
),
origin = candidate.origin,
strictOracleApplicable = candidate.origin == CallsCoverageCandidateOrigin.TERMINATED,
)
appendCallsJournalDurably(path, encodeCallsArtifact(event))
events += event
replayElapsedMillis = replayStarted.elapsedNow().inWholeMilliseconds,
nativeCoverage = callsClassNativeCoverageAcceptance(
original = originalValue,
instrumented = instrumentedValue,
universe = instrumentation.universe,
error = replayError,
),
origin = candidate.origin,
strictOracleApplicable = candidate.origin == CallsCoverageCandidateOrigin.TERMINATED,
)
appendCallsJournalDurably(path, encodeCallsArtifact(event))
events += event
}
}

return events
Expand Down
51 changes: 51 additions & 0 deletions usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsClassScenarioReplayers.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package org.usvm.ts.calls

import org.usvm.ts.pbt.model.JsConcreteValue

/** Reuses only source preparation; every replay still starts a fresh supervised Node.js process. */
internal class CallsClassScenarioReplayers(
private val harness: CallsClassPreparedHarness,
private val instrumentation: CallsClassSourceInstrumentation,
) : AutoCloseable {
private val originalReplayers = mutableMapOf<Int, OriginalTypeScriptClassReplayer>()
private val instrumentedReplayers = mutableMapOf<Int, OriginalTypeScriptClassReplayer>()

fun replay(
scenarioIndex: Int,
inputs: List<JsConcreteValue>,
timeoutMillis: Long,
instrumented: Boolean,
): CallsClassReplayObservation {
val replayers = if (instrumented) instrumentedReplayers else originalReplayers
val replayer = replayers.getOrPut(scenarioIndex) {
OriginalTypeScriptClassReplayer(
sourceRoot = harness.sourceRoot,
function = harness.functions[scenarioIndex],
observationFields = harness.observations[scenarioIndex],
sourceOverride = if (instrumented) instrumentation.instrumentedSource else null,
overrideSourceFile = if (instrumented) harness.surface.sourceFile else null,
counterGlobalName = if (instrumented) instrumentation.hitGlobalName else null,
)
}

return replayer.replay(inputs = inputs, timeoutMillis = timeoutMillis)
}

override fun close() {
var failure: Throwable? = null
for (replayer in originalReplayers.values + instrumentedReplayers.values) {
val error = runCatching { replayer.close() }.exceptionOrNull()
if (error == null) continue

val firstFailure = failure
if (firstFailure == null) {
failure = error
} else {
firstFailure.addSuppressed(error)
}
}
originalReplayers.clear()
instrumentedReplayers.clear()
failure?.let { throw it }
}
}
98 changes: 98 additions & 0 deletions usvm-ts-calls/src/test/kotlin/org/usvm/ts/calls/CallsClassHarnessOracleTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package org.usvm.ts.calls

import org.junit.jupiter.api.io.TempDir
import org.usvm.SolverType
import org.usvm.ts.pbt.fastcheck.PbtBackendException
import org.usvm.ts.pbt.model.ArrayDomain
import org.usvm.ts.pbt.model.ConstantDomain
import org.usvm.ts.pbt.model.IntegerDomain
Expand Down Expand Up @@ -62,6 +63,103 @@ class CallsClassHarnessOracleTest {
}
}

@Test
fun `cached scenario preparation keeps fresh module state inputs and counters for every replay`() {
val statefulSource = "let replayRuns = 0;\n" + source.replace(
"this.count = start;",
"replayRuns++; this.count = start + replayRuns; if (start === 0) this.count += 10;",
)
Files.writeString(directory.resolve("Counter.ts"), statefulSource)
val harness = prepareCallsClassHarness(directory, project(), surface())
val instrumentation = CallsClassSourceInstrumentation.create(
sourceRoot = directory,
sourceFile = "Counter.ts",
className = "Counter",
)

val observations = CallsClassScenarioReplayers(harness, instrumentation).use { replayers ->
listOf(2, 0, 2).map { start ->
val inputs = listOf(number(start))
val original = replayers.replay(
scenarioIndex = 0,
inputs = inputs,
timeoutMillis = 20_000,
instrumented = false,
)
val instrumented = replayers.replay(
scenarioIndex = 0,
inputs = inputs,
timeoutMillis = 20_000,
instrumented = true,
)

assertEquals(inputs, original.inputs)
assertEquals(inputs, instrumented.inputs)
val nativeCoverage = callsClassNativeCoverageAcceptance(
original,
instrumented,
instrumentation.universe,
)
assertTrue(nativeCoverage.accepted)
original to instrumented
}
}

val initialStates = observations.map { it.first.observation.values.getValue("state_0_count") }
assertEquals(listOf(number(3), number(11), number(3)), initialStates)
assertEquals(observations.first().second.coveredStatementKeys, observations.last().second.coveredStatementKeys)
val branchOnlyHits = observations[1].second.coveredStatementKeys - observations[0].second.coveredStatementKeys
assertTrue(branchOnlyHits.isNotEmpty())
}

@Test
fun `cached replay timeout never returns a preceding observation and next input still succeeds`() {
val sourceWithLoop = source.replace(
"this.count = start;",
"if (start === 1) while (true) {} this.count = start;",
)
Files.writeString(directory.resolve("Counter.ts"), sourceWithLoop)
val harness = prepareCallsClassHarness(directory, project(), surface())
val instrumentation = CallsClassSourceInstrumentation.create(
sourceRoot = directory,
sourceFile = "Counter.ts",
className = "Counter",
)

CallsClassScenarioReplayers(harness, instrumentation).use { replayers ->
for (instrumented in listOf(false, true)) {
val first = replayers.replay(
scenarioIndex = 0,
inputs = listOf(number(2)),
timeoutMillis = 20_000,
instrumented = instrumented,
)

val timeout = assertFailsWith<PbtBackendException> {
replayers.replay(
scenarioIndex = 0,
inputs = listOf(number(1)),
timeoutMillis = 50,
instrumented = instrumented,
)
}
assertEquals("backend.process.timeout", timeout.code)

val next = replayers.replay(
scenarioIndex = 0,
inputs = listOf(number(3)),
timeoutMillis = 20_000,
instrumented = instrumented,
)

assertEquals(listOf(number(2)), first.inputs)
assertEquals(number(2), first.observation.values.getValue("state_0_count"))
assertEquals(listOf(number(3)), next.inputs)
assertEquals(number(3), next.observation.values.getValue("state_0_count"))
}
}
}

@Test
fun `object parameter transport stays frozen when callable mutates it`() {
val harness = fixture()
Expand Down
Loading