Mercurial > hg > truffle

comparison truffle/com.oracle.truffle.sl/src/com/oracle/truffle/sl/parser/Scanner.java @ 22190:6050fb3fc842

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
SL: fix rawtypes in Scanner
author Andreas Woess <andreas.woess@oracle.com>
date Wed, 23 Sep 2015 23:38:44 +0200
parents dc83cc1f94f2
children
comparison
equal deleted inserted replaced
22189:fe7032676e60 22190:6050fb3fc842
322 } 322 }
323 323
324 // ----------------------------------------------------------------------------------- 324 // -----------------------------------------------------------------------------------
325 // Scanner 325 // Scanner
326 // ----------------------------------------------------------------------------------- 326 // -----------------------------------------------------------------------------------
327 @SuppressWarnings({"rawtypes", "unchecked"})
328 public class Scanner { 327 public class Scanner {
329 328
330 static final char EOL = '\n'; 329 static final char EOL = '\n';
331 static final int eofSym = 0; 330 static final int eofSym = 0;
332 static final int maxT = 31; 331 static final int maxT = 31;
333 static final int noSym = 31; 332 static final int noSym = 31;
334 333
335 334
336 public Buffer buffer; // scanner buffer 335 public Buffer buffer; // scanner buffer
337 336
338 Token t; // current token 337 Token t; // current token
341 int charPos; // position by unicode characters starting with 0 340 int charPos; // position by unicode characters starting with 0
342 int col; // column number of current character 341 int col; // column number of current character
343 int line; // line number of current character 342 int line; // line number of current character
344 int oldEols; // EOLs that appeared in a comment; 343 int oldEols; // EOLs that appeared in a comment;
345 static final StartStates start; // maps initial token character to start state 344 static final StartStates start; // maps initial token character to start state
346 static final Map literals; // maps literal strings to literal kinds 345 static final Map<String, Integer> literals; // maps literal strings to literal kinds
347 346
348 Token tokens; // list of tokens already peeked (first token is a dummy) 347 Token tokens; // list of tokens already peeked (first token is a dummy)
349 Token pt; // current peek token 348 Token pt; // current peek token
350 349
351 char[] tval = new char[16]; // token text used in NextToken(), dynamically enlarged 350 char[] tval = new char[16]; // token text used in NextToken(), dynamically enlarged
352 int tlen; // length of current token 351 int tlen; // length of current token
353 352
354 static { 353 static {
355 start = new StartStates(); 354 start = new StartStates();
356 literals = new HashMap(); 355 literals = new HashMap<>();
357 for (int i = 65; i <= 90; ++i) start.set(i, 1); 356 for (int i = 65; i <= 90; ++i) start.set(i, 1);
358 for (int i = 97; i <= 122; ++i) start.set(i, 1); 357 for (int i = 97; i <= 122; ++i) start.set(i, 1);
359 for (int i = 49; i <= 57; ++i) start.set(i, 4); 358 for (int i = 49; i <= 57; ++i) start.set(i, 4);
360 start.set(34, 2); 359 start.set(34, 2);
361 start.set(48, 5); 360 start.set(48, 5);
362 start.set(40, 6); 361 start.set(40, 6);
363 start.set(44, 7); 362 start.set(44, 7);
364 start.set(41, 8); 363 start.set(41, 8);
365 start.set(123, 9); 364 start.set(123, 9);
366 start.set(125, 10); 365 start.set(125, 10);
367 start.set(59, 11); 366 start.set(59, 11);
368 start.set(124, 12); 367 start.set(124, 12);
369 start.set(38, 14); 368 start.set(38, 14);
370 start.set(60, 26); 369 start.set(60, 26);
371 start.set(62, 27); 370 start.set(62, 27);
372 start.set(61, 28); 371 start.set(61, 28);
373 start.set(33, 19); 372 start.set(33, 19);
374 start.set(43, 21); 373 start.set(43, 21);
375 start.set(45, 22); 374 start.set(45, 22);
376 start.set(42, 23); 375 start.set(42, 23);
377 start.set(47, 24); 376 start.set(47, 24);
378 start.set(46, 25); 377 start.set(46, 25);
379 start.set(Buffer.EOF, -1); 378 start.set(Buffer.EOF, -1);
380 literals.put("function", new Integer(4)); 379 literals.put("function", new Integer(4));
381 literals.put("break", new Integer(10)); 380 literals.put("break", new Integer(10));
382 literals.put("continue", new Integer(12)); 381 literals.put("continue", new Integer(12));
383 literals.put("while", new Integer(13)); 382 literals.put("while", new Integer(13));
384 literals.put("if", new Integer(14)); 383 literals.put("if", new Integer(14));
385 literals.put("else", new Integer(15)); 384 literals.put("else", new Integer(15));
386 literals.put("return", new Integer(16)); 385 literals.put("return", new Integer(16));
387 386
388 } 387 }
389 388
390 public Scanner(String fileName) { 389 public Scanner(String fileName) {
391 buffer = new Buffer(fileName); 390 buffer = new Buffer(fileName);
447 char[] newBuf = new char[2 * tval.length]; 446 char[] newBuf = new char[2 * tval.length];
448 System.arraycopy(tval, 0, newBuf, 0, tval.length); 447 System.arraycopy(tval, 0, newBuf, 0, tval.length);
449 tval = newBuf; 448 tval = newBuf;
450 } 449 }
451 if (ch != Buffer.EOF) { 450 if (ch != Buffer.EOF) {
452 tval[tlen++] = (char)ch; 451 tval[tlen++] = (char)ch;
453 452
454 NextCh(); 453 NextCh();
455 } 454 }
456 } 455 }
457 456
458 457
459 boolean Comment0() { 458 boolean Comment0() {
460 int level = 1, pos0 = pos, line0 = line, col0 = col, charPos0 = charPos; 459 int level = 1, pos0 = pos, line0 = line, col0 = col, charPos0 = charPos;
461 NextCh(); 460 NextCh();
462 if (ch == '/') { 461 if (ch == '/') {
463 NextCh(); 462 NextCh();
464 for(;;) { 463 for(;;) {
465 if (ch == 10) { 464 if (ch == 10) {
466 level--; 465 level--;
467 if (level == 0) { oldEols = line - line0; NextCh(); return true; } 466 if (level == 0) { oldEols = line - line0; NextCh(); return true; }
468 NextCh(); 467 NextCh();
469 } else if (ch == Buffer.EOF) return false; 468 } else if (ch == Buffer.EOF) return false;
470 else NextCh(); 469 else NextCh();
471 } 470 }
472 } else { 471 } else {
473 buffer.setPos(pos0); NextCh(); line = line0; col = col0; charPos = charPos0; 472 buffer.setPos(pos0); NextCh(); line = line0; col = col0; charPos = charPos0;
474 } 473 }
475 return false; 474 return false;
476 } 475 }
477 476
478 boolean Comment1() { 477 boolean Comment1() {
479 int level = 1, pos0 = pos, line0 = line, col0 = col, charPos0 = charPos; 478 int level = 1, pos0 = pos, line0 = line, col0 = col, charPos0 = charPos;
480 NextCh(); 479 NextCh();
481 if (ch == '*') { 480 if (ch == '*') {
482 NextCh(); 481 NextCh();
483 for(;;) { 482 for(;;) {
484 if (ch == '*') { 483 if (ch == '*') {
485 NextCh(); 484 NextCh();
486 if (ch == '/') { 485 if (ch == '/') {
487 level--; 486 level--;
488 if (level == 0) { oldEols = line - line0; NextCh(); return true; } 487 if (level == 0) { oldEols = line - line0; NextCh(); return true; }
489 NextCh(); 488 NextCh();
490 } 489 }
491 } else if (ch == Buffer.EOF) return false; 490 } else if (ch == Buffer.EOF) return false;
492 else NextCh(); 491 else NextCh();
493 } 492 }
494 } else { 493 } else {
495 buffer.setPos(pos0); NextCh(); line = line0; col = col0; charPos = charPos0; 494 buffer.setPos(pos0); NextCh(); line = line0; col = col0; charPos = charPos0;
496 } 495 }
497 return false; 496 return false;
498 } 497 }
499 498
500 499
501 void CheckLiteral() { 500 void CheckLiteral() {
502 String val = t.val; 501 String val = t.val;
503 502
507 } 506 }
508 } 507 }
509 508
510 Token NextToken() { 509 Token NextToken() {
511 while (ch == ' ' || 510 while (ch == ' ' ||
512 ch >= 9 && ch <= 10 || ch == 13 511 ch >= 9 && ch <= 10 || ch == 13
513 ) NextCh(); 512 ) NextCh();
514 if (ch == '/' && Comment0() ||ch == '/' && Comment1()) return NextToken(); 513 if (ch == '/' && Comment0() ||ch == '/' && Comment1()) return NextToken();
515 int recKind = noSym; 514 int recKind = noSym;
516 int recEnd = pos; 515 int recEnd = pos;
517 t = new Token(); 516 t = new Token();
518 t.pos = pos; 517 t.pos = pos;
519 t.col = col; 518 t.col = col;
535 SetScannerBehindT(); 534 SetScannerBehindT();
536 } 535 }
537 t.kind = recKind; 536 t.kind = recKind;
538 break loop; 537 break loop;
539 } // NextCh already done 538 } // NextCh already done
540 case 1: 539 case 1:
541 recEnd = pos; recKind = 1; 540 recEnd = pos; recKind = 1;
542 if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z') {AddCh(); state = 1; break;} 541 if (ch >= '0' && ch <= '9' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z') {AddCh(); state = 1; break;}
543 else {t.kind = 1; t.val = new String(tval, 0, tlen); CheckLiteral(); return t;} 542 else {t.kind = 1; t.val = new String(tval, 0, tlen); CheckLiteral(); return t;}
544 case 2: 543 case 2:
545 if (ch <= 9 || ch >= 11 && ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '[' || ch >= ']' && ch <= 65535) {AddCh(); state = 2; break;} 544 if (ch <= 9 || ch >= 11 && ch <= 12 || ch >= 14 && ch <= '!' || ch >= '#' && ch <= '[' || ch >= ']' && ch <= 65535) {AddCh(); state = 2; break;}
546 else if (ch == '"') {AddCh(); state = 3; break;} 545 else if (ch == '"') {AddCh(); state = 3; break;}
547 else {state = 0; break;} 546 else {state = 0; break;}
548 case 3: 547 case 3:
549 {t.kind = 2; break loop;} 548 {t.kind = 2; break loop;}
550 case 4: 549 case 4:
551 recEnd = pos; recKind = 3; 550 recEnd = pos; recKind = 3;
552 if (ch >= '0' && ch <= '9') {AddCh(); state = 4; break;} 551 if (ch >= '0' && ch <= '9') {AddCh(); state = 4; break;}
553 else {t.kind = 3; break loop;} 552 else {t.kind = 3; break loop;}
554 case 5: 553 case 5:
555 {t.kind = 3; break loop;} 554 {t.kind = 3; break loop;}
556 case 6: 555 case 6:
557 {t.kind = 5; break loop;} 556 {t.kind = 5; break loop;}
558 case 7: 557 case 7:
559 {t.kind = 6; break loop;} 558 {t.kind = 6; break loop;}
560 case 8: 559 case 8:
561 {t.kind = 7; break loop;} 560 {t.kind = 7; break loop;}
562 case 9: 561 case 9:
563 {t.kind = 8; break loop;} 562 {t.kind = 8; break loop;}
564 case 10: 563 case 10:
565 {t.kind = 9; break loop;} 564 {t.kind = 9; break loop;}
566 case 11: 565 case 11:
567 {t.kind = 11; break loop;} 566 {t.kind = 11; break loop;}
568 case 12: 567 case 12:
569 if (ch == '|') {AddCh(); state = 13; break;} 568 if (ch == '|') {AddCh(); state = 13; break;}
570 else {state = 0; break;} 569 else {state = 0; break;}
571 case 13: 570 case 13:
572 {t.kind = 17; break loop;} 571 {t.kind = 17; break loop;}
573 case 14: 572 case 14:
574 if (ch == '&') {AddCh(); state = 15; break;} 573 if (ch == '&') {AddCh(); state = 15; break;}
575 else {state = 0; break;} 574 else {state = 0; break;}
576 case 15: 575 case 15:
577 {t.kind = 18; break loop;} 576 {t.kind = 18; break loop;}
578 case 16: 577 case 16:
579 {t.kind = 20; break loop;} 578 {t.kind = 20; break loop;}
580 case 17: 579 case 17:
581 {t.kind = 22; break loop;} 580 {t.kind = 22; break loop;}
582 case 18: 581 case 18:
583 {t.kind = 23; break loop;} 582 {t.kind = 23; break loop;}
584 case 19: 583 case 19:
585 if (ch == '=') {AddCh(); state = 20; break;} 584 if (ch == '=') {AddCh(); state = 20; break;}
586 else {state = 0; break;} 585 else {state = 0; break;}
587 case 20: 586 case 20:
588 {t.kind = 24; break loop;} 587 {t.kind = 24; break loop;}
589 case 21: 588 case 21:
590 {t.kind = 25; break loop;} 589 {t.kind = 25; break loop;}
591 case 22: 590 case 22:
592 {t.kind = 26; break loop;} 591 {t.kind = 26; break loop;}
593 case 23: 592 case 23:
594 {t.kind = 27; break loop;} 593 {t.kind = 27; break loop;}
595 case 24: 594 case 24:
596 {t.kind = 28; break loop;} 595 {t.kind = 28; break loop;}
597 case 25: 596 case 25:
598 {t.kind = 30; break loop;} 597 {t.kind = 30; break loop;}
599 case 26: 598 case 26:
600 recEnd = pos; recKind = 19; 599 recEnd = pos; recKind = 19;
601 if (ch == '=') {AddCh(); state = 16; break;} 600 if (ch == '=') {AddCh(); state = 16; break;}
602 else {t.kind = 19; break loop;} 601 else {t.kind = 19; break loop;}
603 case 27: 602 case 27:
604 recEnd = pos; recKind = 21; 603 recEnd = pos; recKind = 21;
605 if (ch == '=') {AddCh(); state = 17; break;} 604 if (ch == '=') {AddCh(); state = 17; break;}
606 else {t.kind = 21; break loop;} 605 else {t.kind = 21; break loop;}
607 case 28: 606 case 28:
608 recEnd = pos; recKind = 29; 607 recEnd = pos; recKind = 29;
609 if (ch == '=') {AddCh(); state = 18; break;} 608 if (ch == '=') {AddCh(); state = 18; break;}
610 else {t.kind = 29; break loop;} 609 else {t.kind = 29; break loop;}
611 610
612 } 611 }
613 } 612 }
614 t.val = new String(tval, 0, tlen); 613 t.val = new String(tval, 0, tlen);
615 return t; 614 return t;