-
Notifications
You must be signed in to change notification settings - Fork 32
/
Fuzzer.fs
301 lines (253 loc) · 12 KB
/
Fuzzer.fs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
namespace VSharp.Fuzzer
open System
open System.Collections.Generic
open System.Diagnostics
open System.IO
open System.Reflection
open System.Threading
open System.Threading.Tasks
open VSharp
open VSharp.CSharpUtils
open VSharp.CoverageTool
open VSharp.Fuzzer.Communication.Contracts
open VSharp.Fuzzer.Startup
open VSharp.Fuzzer.TestGeneration
open Logger
module private CancellableThreads =
let private internalAbort = typeof<System.Runtime.ControlledExecution>.GetMethod("AbortThread", Reflection.allBindingFlags)
let private internalGetThreadHandle = typeof<Thread>.GetMethod("GetNativeHandle", Reflection.allBindingFlags)
let private abortedWasTrying = HashSet<int>()
let mutable private cancellableThreadStartedCount = 0
let private cancellableThreadHandledCount = ref 0
let startCancellableThread f threadId (cancellationToken: CancellationToken) =
cancellableThreadStartedCount <- cancellableThreadStartedCount + 1
let systemThread =
let thread = Thread(fun () -> f(); Interlocked.Increment(cancellableThreadHandledCount) |> ignore)
thread.Start()
thread
let abortThread () =
if systemThread.IsAlive then
abortedWasTrying.Add(threadId) |> ignore
traceFuzzing $"Start aborting: {systemThread.ManagedThreadId}"
let nativeHandle = internalGetThreadHandle.Invoke(systemThread, [||])
internalAbort.Invoke(null, [| nativeHandle |]) |> ignore
systemThread.Join()
traceFuzzing $"Aborted: {systemThread.ManagedThreadId}"
Interlocked.Increment(cancellableThreadHandledCount) |> ignore
Logger.error $"{cancellableThreadStartedCount} = {cancellableThreadHandledCount.Value}"
else
traceFuzzing $"Thread is dead: {systemThread.ManagedThreadId}"
Action abortThread |> cancellationToken.Register |> ignore
systemThread
let wasAbortTried = abortedWasTrying.Contains
let waitAllThreadsHandled () =
let spinner = SpinWait()
while cancellableThreadStartedCount <> cancellableThreadHandledCount.Value do
spinner.SpinOnce()
type internal Fuzzer(
fuzzerOptions: FuzzerOptions,
symbolicExecutionService: IMasterProcessService,
coverageTool: InteractionCoverageTool) =
let rnd = Random(fuzzerOptions.initialSeed)
let typeSolver = TypeSolver()
let generator = Generator(fuzzerOptions, typeSolver)
let threadIdGenerator = Utils.IdGenerator(0)
let testIdGenerator =
let testDir = DirectoryInfo(fuzzerOptions.outputDir)
let tests = testDir.EnumerateFiles("*.vst")
let lastTestId =
tests
|> Seq.map (fun x -> x.Name)
|> Seq.filter (fun x -> x.StartsWith "fuzzer_test_")
|> Seq.map (fun x -> x.Replace("fuzzer_test_", "").Replace(".vst", "") |> int)
|> Seq.cons 0
|> Seq.max
Utils.IdGenerator(lastTestId)
let batchSize = Process.GetCurrentProcess().Threads.Count
let outputDir = fuzzerOptions.outputDir
let mutable generatedCount = 0
let mutable abortedCount = 0
let mutable ignoredCount = 0
let stopwatch = Stopwatch()
let getAvailableTime () =
fuzzerOptions.timeLimitPerMethod - int stopwatch.ElapsedMilliseconds
let copyArgs (args: obj array) =
let wrap o = { object = o }
let unwrap pa = pa.object
let copier = Utils.Copier()
args |> Array.map (wrap >> copier.DeepCopy >> unwrap)
let invoke (method: MethodBase) this args =
try
let returned = method.Invoke(this, copyArgs args)
Returned returned
with
| :? TargetInvocationException as e ->
Thrown e.InnerException
| e ->
logUnhandledException e
let fuzzOnce method (generationDatas: GenerationData[]) (results: InvocationResult[]) i threadId =
fun () ->
coverageTool.SetCurrentThreadId threadId
let generationData = generationDatas[i]
results[i] <- invoke method generationData.this generationData.args
let generateTest (test: UnitTest) =
let testPath = $"{outputDir}{Path.DirectorySeparatorChar}fuzzer_test_{testIdGenerator.NextId()}.vst"
Task.Run(fun () ->
test.Serialize(testPath)
infoFuzzing $"Generated test: {testPath}"
).ForgetUntilExecutionRequested()
let fuzzBatch mockedGenerics typeSolverSeed (rnd: Random) (method: MethodBase) typeStorage =
use fuzzingCancellationTokenSource = new CancellationTokenSource()
traceFuzzing "Generate data"
let data = Array.init batchSize (fun _ -> generator.Generate mockedGenerics method typeStorage (rnd.Next()))
let invocationResults = Array.init batchSize (fun _ -> Unchecked.defaultof<InvocationResult>)
let threadIds = Array.init batchSize (fun _ -> threadIdGenerator.NextId())
traceFuzzing "Data generated"
let indices = [|0..batchSize - 1|]
traceFuzzing "Send execution seeds"
indices
|> Array.iter (fun i ->
symbolicExecutionService.TrackExecutionSeed({
moduleName = method.Module.FullyQualifiedName
methodId = method.MetadataToken
threadId = threadIds[i]
fuzzerSeed = data[i].seed
typeSolverSeed = typeSolverSeed
}).Wait()
)
traceFuzzing "Execution seeds sent"
let availableTime = getAvailableTime ()
if availableTime <= 0 then
traceFuzzing "Method invocation not started, no time available"
None
else
traceFuzzing $"Start method invocation, available time: {availableTime}"
let threads =
indices
|> Array.map (fun i ->
CancellableThreads.startCancellableThread
(fuzzOnce method data invocationResults i threadIds[i])
threadIds[i]
fuzzingCancellationTokenSource.Token
)
fuzzingCancellationTokenSource.CancelAfter(availableTime)
threads |> Array.iter (fun t -> t.Join())
CancellableThreads.waitAllThreadsHandled ()
traceFuzzing "Method invoked"
(threadIds, data, invocationResults) |> Some
let printStatistics (method: Method) =
printfn $"\nMethod[{method.Name}]"
printfn $"Generated: {generatedCount}"
printfn $"Ignored: {ignoredCount}"
printfn $"Aborted: {abortedCount}"
generatedCount <- 0
ignoredCount <- 0
abortedCount <- 0
let handleResults (method: Method) result =
let sendCoverage (methods: Dictionary<int, RawMethodInfo>) coverageReports =
task {
let mainMethod =
methods
|> Seq.find (fun x -> int x.Value.methodToken = method.MetadataToken)
let reports =
coverageReports
|> Array.map (fun coverageReport ->
coverageReport.rawCoverageLocations
|> Array.filter (fun x -> x.methodId = mainMethod.Key)
|> fun x -> {
coverageReport with rawCoverageLocations = x
}
)
let! isNewCoverage = symbolicExecutionService.TrackCoverage({
rawData = reports
methods = methods
})
return isNewCoverage
}
let onNewCoverage generationData invocationResult =
let test = fuzzingResultToTest generationData invocationResult
match test with
| Some test ->
generateTest test
generatedCount <- generatedCount + 1
infoFuzzing "Test will be generated"
| None ->
infoFuzzing "Failed to create test"
ignoredCount <- ignoredCount + 1
()
let onKnownCoverage () =
ignoredCount <- ignoredCount + 1
traceFuzzing "Coverage already tracked"
task {
let (threadIds: int[], data: GenerationData[], invocationResults: InvocationResult[]) = result
let indices = [|0..batchSize - 1|]
traceFuzzing "Coverage requested"
let rawCoverage = coverageTool.GetRawHistory()
traceFuzzing "Coverage received"
let coverages = CoverageDeserializer.getRawReports rawCoverage
traceFuzzing $"Coverage reports[{coverages.reports.Length}] deserialized"
assert (coverages.reports.Length = batchSize)
let reports =
coverages.reports
|> Array.sortBy (fun x -> x.threadId)
|> Array.mapi (fun i x ->
let abortedInsideFuzzerCode =
x.rawCoverageLocations <> [||]
&& Utils.isNull invocationResults[i]
&& CancellableThreads.wasAbortTried x.threadId
if abortedInsideFuzzerCode then
{ x with rawCoverageLocations = [| |] }
else
x
)
let existNonAborted = reports |> Seq.exists (fun x -> x.rawCoverageLocations <> [||])
if existNonAborted then
let! isNewCoverages = sendCoverage coverages.methods reports
for i in indices do
let coverage = reports[i]
let threadId = threadIds[i]
let invocationResult = invocationResults[i]
let generationData = data[i]
let isNewCoverage = isNewCoverages.boolValues[i]
assert (int coverage.threadId = threadId)
traceFuzzing $"Handler result for {threadIds[i]}"
match coverage.rawCoverageLocations with
| [||] ->
traceFuzzing "Aborted"
abortedCount <- abortedCount + 1
| _ ->
traceFuzzing "Invoked"
assert(not <| Utils.isNull invocationResult)
if isNewCoverage then
onNewCoverage generationData invocationResult
else
onKnownCoverage ()
else
traceFuzzing "All aborted"
abortedCount <- abortedCount + reports.Length
}
member this.AsyncFuzz (method: Method) =
task {
try
stopwatch.Reset()
traceFuzzing $"Start fuzzing: {method.Name}, batch size: {batchSize}"
let typeSolverSeed = rnd.Next()
let typeSolverRnd = Random(typeSolverSeed)
match typeSolver.SolveGenericMethodParameters method (generator.GenerateClauseObject typeSolverRnd) with
| Some(methodBase, typeStorage, mockedGenerics) ->
traceFuzzing "Generics successfully solved"
while int stopwatch.ElapsedMilliseconds < fuzzerOptions.timeLimitPerMethod do
traceFuzzing "Start fuzzing iteration"
stopwatch.Start()
match fuzzBatch mockedGenerics typeSolverSeed rnd methodBase typeStorage with
| Some results -> do! handleResults method results
| None -> ()
stopwatch.Stop()
printStatistics method
| None -> traceFuzzing "Generics solving failed"
with
| :? InsufficientInformationException as e ->
errorFuzzing $"Insufficient information: {e.Message}\nStack trace:\n{e.StackTrace}"
| :? NotImplementedException as e ->
errorFuzzing $"Not implemented: {e.Message}\nStack trace:\n{e.StackTrace}"
}