大致实现了词法分析器

include/syntax/Scanner.h

@@ -12,6 +12,24 @@ public:
     Scanner(module_t& module) 
         : module(module) {}
 
+    inline std::vector<token_t> scan() {
+        std::vector<token_t> tokens;
+
+        while (!at_eof()) {
+
+            if (skip_space()) {
+                // 如果是空格或换行，则跳过
+                continue;
+            }
+
+            token_t token = item();
+            tokens.push_back(token);
+        }
+
+        tokens.push_back(token_t(TOKEN_EOF, "EOF", module.s_cursor.line, module.s_cursor.column));
+        return tokens;
+    }
+
     
 private:
     module_t& module;
@@ -90,7 +108,7 @@ private:
     }
 
     inline bool at_eof() {
-        return module.s_cursor.guard == '\0';
+        return module.s_cursor.source[module.s_cursor.guard] == '\0';
     }
 
     inline char guard_advance() {
@@ -127,7 +145,7 @@ private:
         return gen_word();
     }
 
-    inline token_type_t scanner_special_char(module_t *m) {
+    inline token_type_t special_char() {
         char c = guard_advance();
         switch (c) {
             case '(':
@@ -210,6 +228,7 @@ private:
             default:
                 return token_type_t::TOKEN_NOT_IN_THIS_TYPE;
         }
+
     }
 
     inline std::string string_advance() {
@@ -293,6 +312,21 @@ private:
         }
     }
 
+    inline double number_convert_float(std::string word) {
+        try {
+            double decimal = std::stod(word);
+            return decimal;
+        } catch (const std::invalid_argument& e) {
+            std::cerr << "[ERROR] line: " << module.s_cursor.line << ". column: "
+                << module.s_cursor.column <<". Invalid number: " << word << std::endl;
+            return 0;
+        } catch (const std::out_of_range& e) {
+            std::cerr << "[ERROR] line: " << module.s_cursor.line << ". column: "
+                << module.s_cursor.column <<". Number out of range: " << word << std::endl;
+            return 0;
+        }
+    }
+
     inline std::string hex_number_advance() {
         module.s_cursor.guard += 2; // 跳过 0x
 
@@ -300,7 +334,7 @@ private:
             guard_advance();
         }
 
-        return gen_word();
+        return module.s_cursor.source.substr(module.s_cursor.current+2, module.s_cursor.length);
     }
 
     inline std::string oct_number_advance() {
@@ -310,7 +344,7 @@ private:
             guard_advance();
         }
 
-        return gen_word();
+        return module.s_cursor.source.substr(module.s_cursor.current+2, module.s_cursor.length);
     }
 
     inline std::string bin_number_advance() {
@@ -320,7 +354,7 @@ private:
             guard_advance();
         }
 
-        return gen_word();
+        return module.s_cursor.source.substr(module.s_cursor.current+2, module.s_cursor.length);
     }
 
     inline std::string number_advance() {
@@ -358,118 +392,103 @@ private:
             case 'a': {
                 switch (word[1]) {
                     case 'r': {
-                        if (word.substr(2, 3) == "ray") {
+                        if (word.substr(2, 3) == "ray" && word.size() == 5) {
                             return TOKEN_ARR;
                         }
                     }
                 }
                 break;
             }
-            case 'b':
+            case 'b': {
                 switch (word[1]) {
                     case 'o':
-                        if (word.substr(2, 2) == "ol") {
+                        if (word.substr(2, 2) == "ol" && word.size() == 4) {
                             return TOKEN_BOOL;
+                        }
                     case 'r':
-                        if (word.substr(2, 3) == "eak") {
+                        if (word.substr(2, 3) == "eak" && word.size() == 5) {
                             return TOKEN_BREAK;
                         }
                 }
                 break;
-            case 'c':
+            }
+            case 'c': {
                 switch (word[1]) {
                     case 'o':
-                        if (word.substr(2, 6) == "ntinue") {
+                        if (word.substr(2, 6) == "ntinue" && word.size() == 8) {
                             return TOKEN_CONTINUE;
                         }
                 }
                 break;
-            case 'e':
+            }
-                if (word.substr(1, 3) == "lse") {
+            case 'e': {
-                    if (length == 3 && word[3] == 'i') {
+                if (word.substr(1, 3) == "lse" && word.size() == 4) {
-                        return TOKEN_ELSE_IF;
-                    }
                     return TOKEN_ELSE;
                 }
-                return scanner_rest(word, length, 1, 3, "lse", TOKEN_ELSE);
+            }
             case 'f': {
                 switch (word[1]) {
                     case 'n':
-                        return scanner_rest(word, length, 2, 0, "", TOKEN_FN);
+                        if (word.size() == 2) {
+                            return TOKEN_FN;
+                        }
                     case 'a':
-                        return scanner_rest(word, length, 2, 3, "lse", TOKEN_FALSE);
+                        if (word.substr(2, 3) == "lse" && word.size() == 5) {
-                    case 'l':
+                            return TOKEN_FALSE;
-                        return scanner_rest(word, length, 2, 3, "oat", TOKEN_FLOAT);
+                        }
                     case '3':
-                        return scanner_rest(word, length, 2, 1, "2", TOKEN_F32);
+                        if (word.substr(2, 1) == "2" && word.size() == 3) {
+                            return TOKEN_F32;
+                        }
                     case '6':
-                        return scanner_rest(word, length, 2, 1, "4", TOKEN_F64);
+                        if (word.substr(2, 1) == "4" && word.size() == 3) {
+                            return TOKEN_F64;
+                        }
                     case 'o':
-                        return scanner_rest(word, length, 2, 1, "r", TOKEN_FOR);
+                        if (word.substr(2, 1) == "r" && word.size() == 3) {
+                            return TOKEN_FOR;
+                        }
                 }
                 break;
             }
-            case 'g':
-                return scanner_rest(word, length, 1, 1, "o", TOKEN_GO);
             case 'i': {
-                if (length == 2 && word[1] == 'n') {
-                    return TOKEN_IN;
-                } else if (length == 2 && word[1] == 's') {
-                    return TOKEN_IS;
-                } else if (length == 3 && word[1] == 'n' && word[2] == 't') {
-                    return TOKEN_INT;
-                }
-
                 switch (word[1]) {
-                    case 'm':
-                        return scanner_rest(word, length, 2, 4, "port", TOKEN_IMPORT);
                     case 'f':
-                        return scanner_rest(word, length, 2, 0, "", TOKEN_IF);
+                        if (word.size() == 2) {
-                    case 'n':
+                            return TOKEN_IF;
-                        return scanner_rest(word, length, 2, 7, "terface", TOKEN_INTERFACE);
+                        }
                     case '8':
-                        return scanner_rest(word, length, 2, 0, "", TOKEN_I8);
+                        if (word.size() == 2) {
+                            return TOKEN_I8;
+                        }
                     case '1':
-                        return scanner_rest(word, length, 2, 1, "6", TOKEN_I16);
+                        if (word.substr(2, 1) == "6" && word.size() == 3) {
+                            return TOKEN_I16;
+                        }
                     case '3':
-                        return scanner_rest(word, length, 2, 1, "2", TOKEN_I32);
+                        if (word.substr(2, 1) == "2" && word.size() == 3) {
+                            return TOKEN_I32;
+                        }
                     case '6':
-                        return scanner_rest(word, length, 2, 1, "4", TOKEN_I64);
+                        if (word.substr(2, 1) == "4" && word.size() == 3) {
+                            return TOKEN_I64;
+                        }
                 }
                 break;
             }
-            case 'l': {
-                return scanner_rest(word, length, 1, 2, "et", TOKEN_LET);
-            }
-            case 'n':
-                switch (word[1]) {
-                    case 'u': // null
-                        return scanner_rest(word, length, 2, 2, "ll", TOKEN_NULL);
-                        // case 'e':// new, new 识别成 ident 在 parser 采用固定语法结构时才会被识别成 new
-                        // return scanner_rest(word, length, 2, 1, "w", TOKEN_NEW);
-                }
-                break;
-            case 'p':
-                return scanner_rest(word, length, 1, 2, "tr", TOKEN_PTR);
             case 's': {
                 // self,string,struct,sizeof,sett
-                switch (word[1]) {
-                    case 'e': {
-                        switch (word[2]) {
-                            case 't':
-                                return scanner_rest(word, length, 3, 0, "", TOKEN_SET);
-                            case 'l': // select
-                                return scanner_rest(word, length, 3, 3, "ect", TOKEN_SELECT);
-                        }
-                    }
-                }
 
                 if (length == 6 && word[1] == 't' && word[2] == 'r') {
                     switch (word[3]) {
                         case 'i':
-                            return scanner_rest(word, length, 4, 2, "ng", TOKEN_STRING);
+                            if (word.substr(4, 2) == "ng" && word.size() == 6) {
+                                return TOKEN_STRING;
+                            }
                         case 'u':
-                            return scanner_rest(word, length, 4, 2, "ct", TOKEN_STRUCT);
+                            if (word.substr(4, 2) == "ct" && word.size() == 6) {
+                                return TOKEN_STRUCT;
+                            }
                     }
                 }
                 break;
@@ -477,18 +496,20 @@ private:
             case 't': {
                 // tup/throw/type/true
                 switch (word[1]) {
-                    case 'h':
-                        return scanner_rest(word, length, 2, 3, "row", TOKEN_THROW);
                     case 'y': // type
-                        return scanner_rest(word, length, 2, 2, "pe", TOKEN_TYPE);
+                        if (word.substr(2, 2) == "pe" && word.size() == 4) {
+                            return TOKEN_TYPE;
+                        }
                     case 'u': // tup
-                        return scanner_rest(word, length, 2, 1, "p", TOKEN_TUP);
+                        if (word.substr(2, 1) == "p" && word.size() == 3) {
+                            return TOKEN_TUP;
+                        }
                     case 'r': {
                         switch (word[2]) {
-                            case 'y':
-                                return scanner_rest(word, length, 3, 0, "", TOKEN_TRY);
                             case 'u':
-                                return scanner_rest(word, length, 3, 1, "e", TOKEN_TRUE);
+                                if (word.substr(3, 1) == "e" && word.size() == 4) {
+                                    return TOKEN_TRUE;
+                                }
                         }
                         break;
                     }
@@ -498,25 +519,33 @@ private:
             case 'v': {
                 switch (word[1]) {
                     case 'a':
-                        return scanner_rest(word, length, 2, 1, "r", TOKEN_VAR);
+                        if (word.substr(2, 1) == "r" && word.size() == 3) {
+                            return TOKEN_VAR;
+                        }
                     case 'e': // vec
-                        return scanner_rest(word, length, 2, 1, "c", TOKEN_VEC);
+                        if (word.substr(2, 1) == "c" && word.size() == 3) {
-                    case 'o': // void
+                            return TOKEN_VEC;
-                        return scanner_rest(word, length, 2, 2, "id", TOKEN_VOID);
+                        }
                 }
             }
             case 'u': {
                 switch (word[1]) {
-                    case 'i':
-                        return scanner_rest(word, length, 2, 2, "nt", TOKEN_UINT);
                     case '8':
-                        return scanner_rest(word, length, 2, 0, "", TOKEN_U8);
+                        if (word.size() == 2) {
+                            return TOKEN_U8;
+                        }
                     case '1':
-                        return scanner_rest(word, length, 2, 1, "6", TOKEN_U16);
+                        if (word.substr(2, 1) == "6" && word.size() == 3) {
+                            return TOKEN_U16;
+                        }
                     case '3':
-                        return scanner_rest(word, length, 2, 1, "2", TOKEN_U32);
+                        if (word.substr(2, 1) == "2" && word.size() == 3) {
+                            return TOKEN_U32;
+                        }
                     case '6':
-                        return scanner_rest(word, length, 2, 1, "4", TOKEN_U64);
+                        if (word.substr(2, 1) == "4" && word.size() == 3) {
+                            return TOKEN_U64;
+                        }
                 }
                 break;
             }
@@ -526,28 +555,136 @@ private:
                     case 'a': {
                         switch (word[2]) {
                             case 'p':
-                                return scanner_rest(word, length, 3, 0, "", TOKEN_MAP);
+                                if (word.size() == 3) {
-                            case 't':
+                                    return TOKEN_MAP;
-                                return scanner_rest(word, length, 3, 2, "ch", TOKEN_MATCH);
+                                }
                         }
                     }
                 }
             }
             case 'r': {
-                return scanner_rest(word, length, 1, 5, "eturn", TOKEN_RETURN);
+                if (word.substr(1, 5) == "eturn" && word.size() == 6) {
+                    // return
+                    return TOKEN_RETURN;
+                }
             }
         }
 
         return TOKEN_IDENT;
     }
 
+    inline bool multi_comment_end() {
+        return module.s_cursor.source[module.s_cursor.guard] == '*' && 
+               module.s_cursor.source[module.s_cursor.guard + 1] == '/';
+    }
+
+    inline bool skip_space() {
+        bool has_new = false;
+
+        if (module.s_cursor.guard != module.s_cursor.current) {
+            module.s_cursor.space_prev = module.s_cursor.source[module.s_cursor.guard - 1];
+        }
+
+        while (true) {
+            char c = module.s_cursor.source[module.s_cursor.guard];
+            switch (c) {
+                case ' ':
+                case '\r':
+                case '\t': {
+                    guard_advance();
+                    break;
+                }
+                case '\n': {
+                    guard_advance();
+                    has_new = true;
+                    break;
+                }
+                case '/': {
+                    if (module.s_cursor.source[module.s_cursor.guard + 1] == '/') {
+                        // 单行注释
+                        while (module.s_cursor.source[module.s_cursor.guard] != '\n' && !at_eof()) {
+                            guard_advance();
+                        }
+                        break;
+                    } else if (module.s_cursor.source[module.s_cursor.guard + 1] == '*') {
+                        while (!multi_comment_end()) {
+                            if (at_eof()) {
+                                std::cerr << "[ERROR] line: " << module.s_cursor.line << ". column: " 
+                                    << module.s_cursor.column <<". multi comment not end.";
+                                return false;
+                            }
+                            guard_advance();
+                        }
+
+                        guard_advance(); // 跳过 *
+                        guard_advance(); // 跳过 /
+                        break;
+                    } else {
+                        module.s_cursor.space_next = module.s_cursor.source[module.s_cursor.guard];
+                        return has_new;
+                    }
+                    break;
+                }
+                default: {
+                    module.s_cursor.space_next = module.s_cursor.source[module.s_cursor.guard];
+                    return has_new;
+                }
+            }
+        }
+
+    }
+
     inline token_t item() {
         module.s_cursor.length = 0; // 重置长度
-        module.s_cursor.guard = module.s_cursor.current; // 重置游标位置
+        module.s_cursor.current = module.s_cursor.guard; // 重置游标位置
 
         if (is_alpha(module.s_cursor.source[module.s_cursor.guard])) {
             std::string word = ident_advance();
-            return token_t(ident)
+            return token_t(scanner_ident(word, word.size()),word, module.s_cursor.line, module.s_cursor.column);
         }
+
+        if (is_number(module.s_cursor.source[module.s_cursor.guard])) {
+            std::string word;
+            long decimal = 0;
+            
+            if (module.s_cursor.source[module.s_cursor.guard] == '0') {
+                // 可能是十六进制、八进制或二进制
+                if (module.s_cursor.source[module.s_cursor.guard + 1] == 'x' || 
+                    module.s_cursor.source[module.s_cursor.guard + 1] == 'X') {
+                    word = hex_number_advance();
+                    decimal = number_convert(word, 16);
+                } else if (module.s_cursor.source[module.s_cursor.guard + 1] == 'o' || 
+                           module.s_cursor.source[module.s_cursor.guard + 1] == 'O') {
+                    word = oct_number_advance();
+                    decimal = number_convert(word, 8);
+                } else if (module.s_cursor.source[module.s_cursor.guard + 1] == 'b' || 
+                           module.s_cursor.source[module.s_cursor.guard + 1] == 'B') {
+                    word = bin_number_advance();
+                    decimal = number_convert(word, 2);
+                }
+            } else {
+                word = number_advance();
+                decimal = number_convert(word, 10);
+            }
+
+            token_type_t type;
+            if (is_float(word)) {
+                type = TOKEN_LITERAL_FLOAT;
+            } else {
+                type = TOKEN_LITERAL_INT;
+            }
+            return token_t(type, word, module.s_cursor.line, module.s_cursor.column);
+        }
+        if (is_string(module.s_cursor.source[module.s_cursor.guard])) {
+            std::string word = string_advance();
+            return token_t(TOKEN_LITERAL_STRING, word, module.s_cursor.line, module.s_cursor.column);
+        }
+
+        token_type_t type = special_char(); 
+
+        return token_t(type, gen_word(), module.s_cursor.line, module.s_cursor.column);
     }
+
+
 };
+

include/types.hpp

@@ -11,6 +11,9 @@ struct scanner_cursor_t {
 
     int line;  // 扫描器当前所在的行
     int column;   // 扫描器当前所在的列
+
+    char space_prev;  // 记录空行，注释前的上一个字符
+    char space_next;
 };
 
 struct module_t {
@@ -27,5 +30,8 @@ struct module_t {
             s_cursor.length = 0;
             s_cursor.current = 0;
             s_cursor.guard = 0;
+
+            s_cursor.space_prev = '\0';
+            s_cursor.space_next = '\0';
     }
 };

unit/scanner_test.cpp

@@ -1,30 +1,17 @@
-#include "Token.h"
+
 #include "doctest.h"
 #include "stdc++.h"
-#include "Scanner.h"
+#include "types.hpp"
-#include "Tbs.h"
+#include "syntax/Scanner.h"
+#include "syntax/token.h"
 #include <vector>
 using std::string,std::vector;
 
-TEST_CASE("Scanner test identifier table") {
+TEST_CASE("Scanner test") {
-    Tbs tables;
+    module_t module("{}");
-    std::string src = "a += b b<<=casd;";
+    Scanner scanner(module);
-    Scanner scan(src, tables);
-    scan.scan();
 
-    for (auto value_src: scan.get_token_list()) {
+    scanner.scan();
-        std::cout << value_src.id << " " << value_src.type << "\n";
-    }
 }
 
 
-// TEST_CASE("Scanner test Punct table") {
-//     Tbs tables = {};
-//     std::string src = "+=---<<=>>>===--((([]--<<<>.";
-//     Scanner scan(src, tables);
-//     scan.scan();
-//     std::cout<<"test\n";
-//     for (auto e : scan.get_token_list()) {
-//         std::cout<<e.id<<" "<<e.type<<" "<<tables.PunctTable[e.id]<<"\n";
-//     }
-// }