Mercurial > hg > truffle

comparison truffle/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/expression/Scanner.java @ 21951:9c8c0937da41

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Moving all sources into truffle subdirectory
author Jaroslav Tulach <jaroslav.tulach@oracle.com>
date Wed, 17 Jun 2015 10:58:08 +0200
parents graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/expression/Scanner.java@b1530a6cce8c
children dc83cc1f94f2
comparison
equal deleted inserted replaced
21950:2a5011c7e641 21951:9c8c0937da41
1 /*
2 * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 // The content of this file is automatically generated. DO NOT EDIT.
25
26 package com.oracle.truffle.dsl.processor.expression;
27
28 import java.io.*;
29 import java.util.*;
30
31 // Checkstyle: stop
32 // @formatter:off
33 class Token {
34
35 public int kind; // token kind
36 public int pos; // token position in bytes in the source text (starting at 0)
37 public int charPos; // token position in characters in the source text (starting at 0)
38 public int col; // token column (starting at 1)
39 public int line; // token line (starting at 1)
40 public String val; // token value
41 public Token next; // ML 2005-03-11 Peek tokens are kept in linked list
42 }
43
44 // -----------------------------------------------------------------------------------
45 // Buffer
46 // -----------------------------------------------------------------------------------
47 class Buffer {
48
49 // This Buffer supports the following cases:
50 // 1) seekable stream (file)
51 // a) whole stream in buffer
52 // b) part of stream in buffer
53 // 2) non seekable stream (network, console)
54
55 public static final int EOF = Character.MAX_VALUE + 1;
56 private static final int MIN_BUFFER_LENGTH = 1024; // 1KB
57 private static final int MAX_BUFFER_LENGTH = MIN_BUFFER_LENGTH * 64; // 64KB
58 private byte[] buf; // input buffer
59 private int bufStart; // position of first byte in buffer relative to input stream
60 private int bufLen; // length of buffer
61 private int fileLen; // length of input stream (may change if stream is no file)
62 private int bufPos; // current position in buffer
63 private RandomAccessFile file; // input stream (seekable)
64 private InputStream stream; // growing input stream (e.g.: console, network)
65
66 public Buffer(InputStream s) {
67 stream = s;
68 fileLen = bufLen = bufStart = bufPos = 0;
69 buf = new byte[MIN_BUFFER_LENGTH];
70 }
71
72 public Buffer(String fileName) {
73 try {
74 file = new RandomAccessFile(fileName, "r");
75 fileLen = (int) file.length();
76 bufLen = Math.min(fileLen, MAX_BUFFER_LENGTH);
77 buf = new byte[bufLen];
78 bufStart = Integer.MAX_VALUE; // nothing in buffer so far
79 if (fileLen > 0)
80 setPos(0); // setup buffer to position 0 (start)
81 else
82 bufPos = 0; // index 0 is already after the file, thus setPos(0) is invalid
83 if (bufLen == fileLen)
84 Close();
85 } catch (IOException e) {
86 throw new FatalError("Could not open file " + fileName);
87 }
88 }
89
90 // don't use b after this call anymore
91 // called in UTF8Buffer constructor
92 protected Buffer(Buffer b) {
93 buf = b.buf;
94 bufStart = b.bufStart;
95 bufLen = b.bufLen;
96 fileLen = b.fileLen;
97 bufPos = b.bufPos;
98 file = b.file;
99 stream = b.stream;
100 // keep finalize from closing the file
101 b.file = null;
102 }
103
104 @Override
105 protected void finalize() throws Throwable {
106 super.finalize();
107 Close();
108 }
109
110 protected void Close() {
111 if (file != null) {
112 try {
113 file.close();
114 file = null;
115 } catch (IOException e) {
116 throw new FatalError(e.getMessage());
117 }
118 }
119 }
120
121 public int Read() {
122 if (bufPos < bufLen) {
123 return buf[bufPos++] & 0xff; // mask out sign bits
124 } else if (getPos() < fileLen) {
125 setPos(getPos()); // shift buffer start to pos
126 return buf[bufPos++] & 0xff; // mask out sign bits
127 } else if (stream != null && ReadNextStreamChunk() > 0) {
128 return buf[bufPos++] & 0xff; // mask out sign bits
129 } else {
130 return EOF;
131 }
132 }
133
134 public int Peek() {
135 int curPos = getPos();
136 int ch = Read();
137 setPos(curPos);
138 return ch;
139 }
140
141 // beg .. begin, zero-based, inclusive, in byte
142 // end .. end, zero-based, exclusive, in byte
143 public String GetString(int beg, int end) {
144 int len = 0;
145 char[] buffer = new char[end - beg];
146 int oldPos = getPos();
147 setPos(beg);
148 while (getPos() < end)
149 buffer[len++] = (char) Read();
150 setPos(oldPos);
151 return new String(buffer, 0, len);
152 }
153
154 public int getPos() {
155 return bufPos + bufStart;
156 }
157
158 public void setPos(int value) {
159 if (value >= fileLen && stream != null) {
160 // Wanted position is after buffer and the stream
161 // is not seek-able e.g. network or console,
162 // thus we have to read the stream manually till
163 // the wanted position is in sight.
164 while (value >= fileLen && ReadNextStreamChunk() > 0) {
165 // nothing to do...
166 }
167 }
168
169 if (value < 0 || value > fileLen) {
170 throw new FatalError("buffer out of bounds access, position: " + value);
171 }
172
173 if (value >= bufStart && value < bufStart + bufLen) { // already in buffer
174 bufPos = value - bufStart;
175 } else if (file != null) { // must be swapped in
176 try {
177 file.seek(value);
178 bufLen = file.read(buf);
179 bufStart = value;
180 bufPos = 0;
181 } catch (IOException e) {
182 throw new FatalError(e.getMessage());
183 }
184 } else {
185 // set the position to the end of the file, Pos will return fileLen.
186 bufPos = fileLen - bufStart;
187 }
188 }
189
190 // Read the next chunk of bytes from the stream, increases the buffer
191 // if needed and updates the fields fileLen and bufLen.
192 // Returns the number of bytes read.
193 private int ReadNextStreamChunk() {
194 int free = buf.length - bufLen;
195 if (free == 0) {
196 // in the case of a growing input stream
197 // we can neither seek in the stream, nor can we
198 // foresee the maximum length, thus we must adapt
199 // the buffer size on demand.
200 byte[] newBuf = new byte[bufLen * 2];
201 System.arraycopy(buf, 0, newBuf, 0, bufLen);
202 buf = newBuf;
203 free = bufLen;
204 }
205
206 int read;
207 try {
208 read = stream.read(buf, bufLen, free);
209 } catch (IOException ioex) {
210 throw new FatalError(ioex.getMessage());
211 }
212
213 if (read > 0) {
214 fileLen = bufLen = (bufLen + read);
215 return read;
216 }
217 // end of stream reached
218 return 0;
219 }
220 }
221
222 // -----------------------------------------------------------------------------------
223 // UTF8Buffer
224 // -----------------------------------------------------------------------------------
225 class UTF8Buffer extends Buffer {
226
227 UTF8Buffer(Buffer b) {
228 super(b);
229 }
230
231 @Override
232 public int Read() {
233 int ch;
234 do {
235 ch = super.Read();
236 // until we find a utf8 start (0xxxxxxx or 11xxxxxx)
237 } while ((ch >= 128) && ((ch & 0xC0) != 0xC0) && (ch != EOF));
238 if (ch < 128 || ch == EOF) {
239 // nothing to do, first 127 chars are the same in ascii and utf8
240 // 0xxxxxxx or end of file character
241 } else if ((ch & 0xF0) == 0xF0) {
242 // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
243 int c1 = ch & 0x07;
244 ch = super.Read();
245 int c2 = ch & 0x3F;
246 ch = super.Read();
247 int c3 = ch & 0x3F;
248 ch = super.Read();
249 int c4 = ch & 0x3F;
250 ch = (((((c1 << 6) | c2) << 6) | c3) << 6) | c4;
251 } else if ((ch & 0xE0) == 0xE0) {
252 // 1110xxxx 10xxxxxx 10xxxxxx
253 int c1 = ch & 0x0F;
254 ch = super.Read();
255 int c2 = ch & 0x3F;
256 ch = super.Read();
257 int c3 = ch & 0x3F;
258 ch = (((c1 << 6) | c2) << 6) | c3;
259 } else if ((ch & 0xC0) == 0xC0) {
260 // 110xxxxx 10xxxxxx
261 int c1 = ch & 0x1F;
262 ch = super.Read();
263 int c2 = ch & 0x3F;
264 ch = (c1 << 6) | c2;
265 }
266 return ch;
267 }
268 }
269
270 // -----------------------------------------------------------------------------------
271 // StartStates -- maps characters to start states of tokens
272 // -----------------------------------------------------------------------------------
273 class StartStates {
274
275 private static class Elem {
276
277 public int key, val;
278 public Elem next;
279
280 public Elem(int key, int val) {
281 this.key = key;
282 this.val = val;
283 }
284 }
285
286 private Elem[] tab = new Elem[128];
287
288 public void set(int key, int val) {
289 Elem e = new Elem(key, val);
290 int k = key % 128;
291 e.next = tab[k];
292 tab[k] = e;
293 }
294
295 public int state(int key) {
296 Elem e = tab[key % 128];
297 while (e != null && e.key != key)
298 e = e.next;
299 return e == null ? 0 : e.val;
300 }
301 }
302
303 // -----------------------------------------------------------------------------------
304 // Scanner
305 // -----------------------------------------------------------------------------------
306 @SuppressWarnings({"rawtypes"})
307 public class Scanner {
308
309 static final char EOL = '\n';
310 static final int eofSym = 0;
311 static final int maxT = 15;
312 static final int noSym = 15;
313
314
315 public Buffer buffer; // scanner buffer
316
317 Token t; // current token
318 int ch; // current input character
319 int pos; // byte position of current character
320 int charPos; // position by unicode characters starting with 0
321 int col; // column number of current character
322 int line; // line number of current character
323 int oldEols; // EOLs that appeared in a comment;
324 static final StartStates start; // maps initial token character to start state
325 static final Map literals; // maps literal strings to literal kinds
326
327 Token tokens; // list of tokens already peeked (first token is a dummy)
328 Token pt; // current peek token
329
330 char[] tval = new char[16]; // token text used in NextToken(), dynamically enlarged
331 int tlen; // length of current token
332
333 static {
334 start = new StartStates();
335 literals = new HashMap();
336 for (int i = 36; i <= 36; ++i) start.set(i, 1);
337 for (int i = 65; i <= 90; ++i) start.set(i, 1);
338 for (int i = 95; i <= 95; ++i) start.set(i, 1);
339 for (int i = 97; i <= 122; ++i) start.set(i, 1);
340 for (int i = 49; i <= 57; ++i) start.set(i, 6);
341 start.set(48, 2);
342 start.set(124, 7);
343 start.set(60, 18);
344 start.set(62, 19);
345 start.set(61, 11);
346 start.set(33, 20);
347 start.set(40, 14);
348 start.set(41, 15);
349 start.set(44, 16);
350 start.set(46, 17);
351 start.set(Buffer.EOF, -1);
352
353 }
354
355 public Scanner(String fileName) {
356 buffer = new Buffer(fileName);
357 Init();
358 }
359
360 public Scanner(InputStream s) {
361 buffer = new Buffer(s);
362 Init();
363 }
364
365 void Init() {
366 pos = -1;
367 line = 1;
368 col = 0;
369 charPos = -1;
370 oldEols = 0;
371 NextCh();
372 if (ch == 0xEF) { // check optional byte order mark for UTF-8
373 NextCh();
374 int ch1 = ch;
375 NextCh();
376 int ch2 = ch;
377 if (ch1 != 0xBB || ch2 != 0xBF) {
378 throw new FatalError("Illegal byte order mark at start of file");
379 }
380 buffer = new UTF8Buffer(buffer);
381 col = 0;
382 charPos = -1;
383 NextCh();
384 }
385 pt = tokens = new Token(); // first token is a dummy
386 }
387
388 void NextCh() {
389 if (oldEols > 0) {
390 ch = EOL;
391 oldEols--;
392 } else {
393 pos = buffer.getPos();
394 // buffer reads unicode chars, if UTF8 has been detected
395 ch = buffer.Read();
396 col++;
397 charPos++;
398 // replace isolated '\r' by '\n' in order to make
399 // eol handling uniform across Windows, Unix and Mac
400 if (ch == '\r' && buffer.Peek() != '\n')
401 ch = EOL;
402 if (ch == EOL) {
403 line++;
404 col = 0;
405 }
406 }
407
408 }
409
410 void AddCh() {
411 if (tlen >= tval.length) {
412 char[] newBuf = new char[2 * tval.length];
413 System.arraycopy(tval, 0, newBuf, 0, tval.length);
414 tval = newBuf;
415 }
416 if (ch != Buffer.EOF) {
417 tval[tlen++] = (char)ch;
418
419 NextCh();
420 }
421 }
422
423
424
425 void CheckLiteral() {
426 String val = t.val;
427
428 Object kind = literals.get(val);
429 if (kind != null) {
430 t.kind = ((Integer) kind).intValue();
431 }
432 }
433
434 Token NextToken() {
435 while (ch == ' ' ||
436 false
437 ) NextCh();
438
439 int recKind = noSym;
440 int recEnd = pos;
441 t = new Token();
442 t.pos = pos;
443 t.col = col;
444 t.line = line;
445 t.charPos = charPos;
446 int state = start.state(ch);
447 tlen = 0;
448 AddCh();
449
450 loop: for (;;) {
451 switch (state) {
452 case -1: {
453 t.kind = eofSym;
454 break loop;
455 } // NextCh already done
456 case 0: {
457 if (recKind != noSym) {
458 tlen = recEnd - t.pos;
459 SetScannerBehindT();
460 }
461 t.kind = recKind;
462 break loop;
463 } // NextCh already done
464 case 1:
465 recEnd = pos; recKind = 1;
466 if (ch == '$' || ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'Z' || ch == '_' || ch >= 'a' && ch <= 'z') {AddCh(); state = 1; break;}
467 else {t.kind = 1; break loop;}
468 case 2:
469 recEnd = pos; recKind = 2;
470 if (ch >= '0' && ch <= '7') {AddCh(); state = 5; break;}
471 else if (ch == 'x') {AddCh(); state = 3; break;}
472 else if (ch == 'b') {AddCh(); state = 4; break;}
473 else {t.kind = 2; break loop;}
474 case 3:
475 recEnd = pos; recKind = 2;
476 if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'F' || ch >= 'a' && ch <= 'f') {AddCh(); state = 3; break;}
477 else {t.kind = 2; break loop;}
478 case 4:
479 recEnd = pos; recKind = 2;
480 if (ch >= '0' && ch <= '1') {AddCh(); state = 4; break;}
481 else {t.kind = 2; break loop;}
482 case 5:
483 recEnd = pos; recKind = 2;
484 if (ch >= '0' && ch <= '7') {AddCh(); state = 5; break;}
485 else {t.kind = 2; break loop;}
486 case 6:
487 recEnd = pos; recKind = 2;
488 if (ch >= '0' && ch <= '9') {AddCh(); state = 6; break;}
489 else {t.kind = 2; break loop;}
490 case 7:
491 if (ch == '|') {AddCh(); state = 8; break;}
492 else {state = 0; break;}
493 case 8:
494 {t.kind = 3; break loop;}
495 case 9:
496 {t.kind = 5; break loop;}
497 case 10:
498 {t.kind = 7; break loop;}
499 case 11:
500 if (ch == '=') {AddCh(); state = 12; break;}
501 else {state = 0; break;}
502 case 12:
503 {t.kind = 8; break loop;}
504 case 13:
505 {t.kind = 9; break loop;}
506 case 14:
507 {t.kind = 11; break loop;}
508 case 15:
509 {t.kind = 12; break loop;}
510 case 16:
511 {t.kind = 13; break loop;}
512 case 17:
513 {t.kind = 14; break loop;}
514 case 18:
515 recEnd = pos; recKind = 4;
516 if (ch == '=') {AddCh(); state = 9; break;}
517 else {t.kind = 4; break loop;}
518 case 19:
519 recEnd = pos; recKind = 6;
520 if (ch == '=') {AddCh(); state = 10; break;}
521 else {t.kind = 6; break loop;}
522 case 20:
523 recEnd = pos; recKind = 10;
524 if (ch == '=') {AddCh(); state = 13; break;}
525 else {t.kind = 10; break loop;}
526
527 }
528 }
529 t.val = new String(tval, 0, tlen);
530 return t;
531 }
532
533 private void SetScannerBehindT() {
534 buffer.setPos(t.pos);
535 NextCh();
536 line = t.line;
537 col = t.col;
538 charPos = t.charPos;
539 for (int i = 0; i < tlen; i++)
540 NextCh();
541 }
542
543 // get the next token (possibly a token already seen during peeking)
544 public Token Scan() {
545 if (tokens.next == null) {
546 return NextToken();
547 } else {
548 pt = tokens = tokens.next;
549 return tokens;
550 }
551 }
552
553 // get the next token, ignore pragmas
554 public Token Peek() {
555 do {
556 if (pt.next == null) {
557 pt.next = NextToken();
558 }
559 pt = pt.next;
560 } while (pt.kind > maxT); // skip pragmas
561
562 return pt;
563 }
564
565 // make sure that peeking starts at current scan position
566 public void ResetPeek() {
567 pt = tokens;
568 }
569
570 } // end Scanner