Mercurial > hg > truffle

comparison graal/com.oracle.truffle.sl.test/src/com/oracle/truffle/sl/test/instrument/SLInstrumentTestRunner.java @ 16702:2a5ec181dad4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
SL: Added instrumentation testing
author David Piorkowski <david.piorkowski@oracle.com>
date Tue, 05 Aug 2014 16:34:08 -0700
parents
children a24beb9c9993
comparison
equal deleted inserted replaced
16696:bd6b44b04143 16702:2a5ec181dad4
1 /*
2 * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package com.oracle.truffle.sl.test.instrument;
24
25 import java.io.*;
26 import java.nio.charset.*;
27 import java.nio.file.*;
28 import java.nio.file.attribute.*;
29 import java.util.*;
30
31 import org.junit.*;
32 import org.junit.internal.*;
33 import org.junit.runner.*;
34 import org.junit.runner.manipulation.*;
35 import org.junit.runner.notification.*;
36 import org.junit.runners.*;
37 import org.junit.runners.model.*;
38
39 import com.oracle.truffle.api.*;
40 import com.oracle.truffle.api.nodes.*;
41 import com.oracle.truffle.api.source.*;
42 import com.oracle.truffle.sl.nodes.instrument.*;
43 import com.oracle.truffle.sl.parser.*;
44 import com.oracle.truffle.sl.runtime.*;
45 import com.oracle.truffle.sl.test.*;
46 import com.oracle.truffle.sl.test.instrument.SLInstrumentTestRunner.InstrumentTestCase;
47
48 /**
49 * This class builds and executes the tests for instrumenting SL. Although much of this class is
50 * written with future automation in mind, at the moment the tests that are created are hard-coded
51 * according to the file name of the test. To be automated, an automatic way of generating both the
52 * node visitor and the node prober is necessary.
53 *
54 * Testing is done via JUnit via comparing execution outputs with expected outputs.
55 */
56 public final class SLInstrumentTestRunner extends ParentRunner<InstrumentTestCase> {
57
58 private static final String SOURCE_SUFFIX = ".sl";
59 private static final String INPUT_SUFFIX = ".input";
60 private static final String OUTPUT_SUFFIX = ".output";
61 private static final String VISITOR_ASSIGNMENT_COUNT_SUFFIX = "_assnCount";
62 private static final String VISITOR_VARIABLE_COMPARE_SUFFIX = "_varCompare";
63
64 private static final String LF = System.getProperty("line.separator");
65 private static SLContext slContext;
66
67 static class InstrumentTestCase {
68 protected final Description name;
69 protected final Path path;
70 protected final String baseName;
71 protected final String sourceName;
72 protected final String testInput;
73 protected final String expectedOutput;
74 protected String actualOutput;
75
76 protected InstrumentTestCase(Class<?> testClass, String baseName, String sourceName, Path path, String testInput, String expectedOutput) {
77 this.name = Description.createTestDescription(testClass, baseName);
78 this.baseName = baseName;
79 this.sourceName = sourceName;
80 this.path = path;
81 this.testInput = testInput;
82 this.expectedOutput = expectedOutput;
83 }
84 }
85
86 private final List<InstrumentTestCase> testCases;
87
88 public SLInstrumentTestRunner(Class<?> testClass) throws InitializationError {
89 super(testClass);
90 try {
91 testCases = createTests(testClass);
92 } catch (IOException e) {
93 throw new InitializationError(e);
94 }
95 }
96
97 @Override
98 protected List<InstrumentTestCase> getChildren() {
99 return testCases;
100 }
101
102 @Override
103 protected Description describeChild(InstrumentTestCase child) {
104 return child.name;
105 }
106
107 /**
108 * Tests are created based on the files that exist in the directory specified in the passed in
109 * annotation. Each test must have a source file and an expected output file. Optionally, each
110 * test can also include an input file. Source files have an ".sl" extension. Expected output
111 * have a ".output" extension. Input files have an ".input" extension. All these files must
112 * share the same base name to be correctly grouped. For example: "test1_assnCount.sl",
113 * "test1_assnCount.output" and "test1_assnCount.input" would all be used to create a single
114 * test called "test1_assnCount".
115 *
116 * This method iterates over the files in the directory and creates a new InstrumentTestCase for
117 * each group of related files. Each file is also expected to end with an identified at the end
118 * of the base name to indicate what visitor needs to be attached. Currently, visitors are hard
119 * coded to work on specific lines, so the code here is not currently generalizable.
120 *
121 * @param c The annotation containing the directory with tests
122 * @return A list of {@link InstrumentTestCase}s to run.
123 * @throws IOException If the directory is invalid.
124 * @throws InitializationError If no directory is provided.
125 *
126 * @see #runChild(InstrumentTestCase, RunNotifier)
127 */
128 protected static List<InstrumentTestCase> createTests(final Class<?> c) throws IOException, InitializationError {
129 SLInstrumentTestSuite suite = c.getAnnotation(SLInstrumentTestSuite.class);
130 if (suite == null) {
131 throw new InitializationError(String.format("@%s annotation required on class '%s' to run with '%s'.", SLTestSuite.class.getSimpleName(), c.getName(), SLTestRunner.class.getSimpleName()));
132 }
133
134 String[] paths = suite.value();
135
136 Path root = null;
137 for (String path : paths) {
138 root = FileSystems.getDefault().getPath(path);
139 if (Files.exists(root)) {
140 break;
141 }
142 }
143 if (root == null && paths.length > 0) {
144 throw new FileNotFoundException(paths[0]);
145 }
146
147 final Path rootPath = root;
148
149 final List<InstrumentTestCase> testCases = new ArrayList<>();
150
151 // Scaffolding in place for future automation
152 Files.walkFileTree(rootPath, new SimpleFileVisitor<Path>() {
153 @Override
154 public FileVisitResult visitFile(Path sourceFile, BasicFileAttributes attrs) throws IOException {
155 String sourceName = sourceFile.getFileName().toString();
156 if (sourceName.endsWith(SOURCE_SUFFIX)) {
157 String baseName = sourceName.substring(0, sourceName.length() - SOURCE_SUFFIX.length());
158
159 Path inputFile = sourceFile.resolveSibling(baseName + INPUT_SUFFIX);
160 String testInput = "";
161 if (Files.exists(inputFile)) {
162 testInput = readAllLines(inputFile);
163 }
164
165 Path outputFile = sourceFile.resolveSibling(baseName + OUTPUT_SUFFIX);
166 String expectedOutput = "";
167 if (Files.exists(outputFile)) {
168 expectedOutput = readAllLines(outputFile);
169 }
170
171 testCases.add(new InstrumentTestCase(c, baseName, sourceName, sourceFile, testInput, expectedOutput));
172
173 }
174 return FileVisitResult.CONTINUE;
175 }
176 });
177
178 return testCases;
179 }
180
181 private static String readAllLines(Path file) throws IOException {
182 // fix line feeds for non unix os
183 StringBuilder outFile = new StringBuilder();
184 for (String line : Files.readAllLines(file, Charset.defaultCharset())) {
185 outFile.append(line).append(LF);
186 }
187 return outFile.toString();
188 }
189
190 /**
191 * Executes the passed in test case. Instrumentation is added according to the name of the file
192 * as explained in {@link #createTests(Class)}. Note that this code is not generalizable.
193 */
194 @Override
195 protected void runChild(InstrumentTestCase testCase, RunNotifier notifier) {
196 // TODO Current tests are hard-coded, automate this eventually
197 notifier.fireTestStarted(testCase.name);
198
199 ByteArrayOutputStream out = new ByteArrayOutputStream();
200 PrintStream printer = new PrintStream(out);
201 try {
202 // We use the name of the file to determine what visitor to attach to it.
203 if (testCase.baseName.endsWith(VISITOR_ASSIGNMENT_COUNT_SUFFIX) || testCase.baseName.endsWith(VISITOR_VARIABLE_COMPARE_SUFFIX)) {
204 NodeVisitor nodeVisitor = null;
205 slContext = new SLContext(new BufferedReader(new StringReader(testCase.testInput)), printer);
206 final Source source = Source.fromText(readAllLines(testCase.path), testCase.sourceName);
207 SLASTProber prober = new SLASTProber();
208
209 // Note that the visitor looks for an attachment point via line number
210 if (testCase.baseName.endsWith(VISITOR_ASSIGNMENT_COUNT_SUFFIX)) {
211 nodeVisitor = new NodeVisitor() {
212
213 public boolean visit(Node node) {
214 if (node instanceof SLExpressionWrapper) {
215 SLExpressionWrapper wrapper = (SLExpressionWrapper) node;
216 int lineNum = wrapper.getSourceSection().getLineLocation().getLineNumber();
217
218 if (lineNum == 4) {
219 wrapper.getProbe().addInstrument(new SLPrintAssigmentValueInstrument(slContext.getOutput()));
220 }
221 }
222 return true;
223 }
224 };
225
226 // Note that the visitor looks for an attachment point via line number
227 } else if (testCase.baseName.endsWith(VISITOR_VARIABLE_COMPARE_SUFFIX)) {
228 nodeVisitor = new NodeVisitor() {
229
230 public boolean visit(Node node) {
231 if (node instanceof SLStatementWrapper) {
232 SLStatementWrapper wrapper = (SLStatementWrapper) node;
233 int lineNum = wrapper.getSourceSection().getLineLocation().getLineNumber();
234
235 if (lineNum == 6) {
236 wrapper.getProbe().addInstrument(new SLCheckVariableEqualityInstrument("i", "count", slContext.getOutput()));
237 }
238 }
239 return true;
240 }
241 };
242 }
243
244 prober.addNodeProber(new SLInstrumentTestNodeProber(slContext));
245 Parser.parseSL(slContext, source, prober);
246 List<SLFunction> functionList = slContext.getFunctionRegistry().getFunctions();
247
248 // Since only functions can be global in SL, this guarantees that we instrument
249 // everything of interest. Parsing must occur before accepting the visitors since
250 // parsing is what creates our instrumentation points.
251 for (SLFunction function : functionList) {
252 RootCallTarget rootCallTarget = function.getCallTarget();
253 rootCallTarget.getRootNode().accept(nodeVisitor);
254 }
255
256 SLFunction main = slContext.getFunctionRegistry().lookup("main");
257 main.getCallTarget().call();
258 } else {
259 notifier.fireTestFailure(new Failure(testCase.name, new UnsupportedOperationException("No instrumentation found.")));
260 }
261
262 String actualOutput = new String(out.toByteArray());
263 Assert.assertEquals(testCase.expectedOutput, actualOutput);
264 } catch (Throwable ex) {
265 notifier.fireTestFailure(new Failure(testCase.name, ex));
266 } finally {
267 notifier.fireTestFinished(testCase.name);
268 }
269
270 }
271
272 public static void runInMain(Class<?> testClass, String[] args) throws InitializationError, NoTestsRemainException {
273 JUnitCore core = new JUnitCore();
274 core.addListener(new TextListener(System.out));
275 SLTestRunner suite = new SLTestRunner(testClass);
276 if (args.length > 0) {
277 suite.filter(new NameFilter(args[0]));
278 }
279 Result r = core.run(suite);
280 if (!r.wasSuccessful()) {
281 System.exit(1);
282 }
283 }
284
285 private static final class NameFilter extends Filter {
286 private final String pattern;
287
288 private NameFilter(String pattern) {
289 this.pattern = pattern.toLowerCase();
290 }
291
292 @Override
293 public boolean shouldRun(Description description) {
294 return description.getMethodName().toLowerCase().contains(pattern);
295 }
296
297 @Override
298 public String describe() {
299 return "Filter contains " + pattern;
300 }
301 }
302 }