#pragma once #include "doctest.h" #include "token.h" #include "../types.hpp" #include #include #include #include class Scanner { public: Scanner(module_t& module) : module(module) {} private: module_t& module; inline std::string gen_word() { return module.s_cursor.source.substr(module.s_cursor.current, module.s_cursor.length); } inline bool is_space(char c) { if (c == '\n' || c == '\t' || c == '\r' || c == ' ') { return true; } return false; } inline bool is_string(char s) { return s == '"'; } inline bool is_float(std::string word) { // 是否包含 .,包含则为 float int dot_count = 0; bool has_e = false; for (std::string::size_type i = 0; i < word.size(); i++) { if (word[i] == '.') dot_count++; else if (word[i] == 'e' || word[i] == 'E') has_e = true; } // 结尾不能是 . if (word[-1] == '.') { std::cerr << "[ERROR] line: " << module.s_cursor.line << ". column: " << module.s_cursor.column <<". floating-point numbers cannot end with '.'"; return false; } // 如果有科学计数法标记,则认为是浮点数 if (has_e) { return true; } if (dot_count == 0) { return false; } if (dot_count > 1) { std::cerr << "[ERROR] line: " << module.s_cursor.line << ". column: " << module.s_cursor.column <<". floating-point numbers have multiple '.'"; return false; } return true; } inline bool is_alpha(char c) { return std::isalpha(c); } inline bool is_number(char c) { return std::isdigit(c); } inline bool is_hex_number(char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } inline bool is_oct_number(char c) { return c >= '0' && c <= '7'; } inline bool is_bin_number(char c) { return c == '0' || c == '1'; } inline bool at_eof() { return module.s_cursor.guard == '\0'; } inline char guard_advance() { module.s_cursor.guard++; module.s_cursor.length++; module.s_cursor.column++; if (module.s_cursor.source[module.s_cursor.guard] == '\n') { module.s_cursor.line++; module.s_cursor.column = 0; } return module.s_cursor.source[module.s_cursor.guard]; } inline bool match(char expected) { if (at_eof()) return false; if (module.source[module.s_cursor.guard] != expected) return false; guard_advance(); return true; } inline std::string ident_advance() { while((is_alpha(module.s_cursor.source[module.s_cursor.guard]) || is_number(module.s_cursor.source[module.s_cursor.guard])) && !at_eof()) { guard_advance(); } return gen_word(); } inline token_type_t scanner_special_char(module_t *m) { char c = guard_advance(); switch (c) { case '(': return TOKEN_LEFT_PAREN; case ')': return TOKEN_RIGHT_PAREN; case '[': return TOKEN_LEFT_SQUARE; case ']': return TOKEN_RIGHT_SQUARE; case '{': return TOKEN_LEFT_CURLY; case '}': return TOKEN_RIGHT_CURLY; case ':': return TOKEN_COLON; case ';': return TOKEN_STMT_EOF; case ',': return TOKEN_COMMA; case '?': return TOKEN_QUESTION; case '%': return match('=') ? TOKEN_PERSON_EQUAL : TOKEN_PERSON; case '-': if (match('=')) { return TOKEN_MINUS_EQUAL; } if (match('>')) { return TOKEN_RIGHT_ARROW; } return TOKEN_MINUS; case '+': return match('=') ? TOKEN_PLUS_EQUAL : TOKEN_PLUS; case '/': return match('=') ? TOKEN_SLASH_EQUAL : TOKEN_SLASH; case '*': { return match('=') ? TOKEN_STAR_EQUAL : TOKEN_STAR; } case '.': { return TOKEN_DOT; } case '!': return match('=') ? TOKEN_NOT_EQUAL : TOKEN_NOT; case '=': return match('=') ? TOKEN_EQUAL_EQUAL : TOKEN_EQUAL; case '<': if (match('<')) { // << if (match('=')) { // <<= return TOKEN_LEFT_SHIFT_EQUAL; } // << return TOKEN_LEFT_SHIFT; } else if (match('=')) { return TOKEN_LESS_EQUAL; } return TOKEN_LEFT_ANGLE; case '>': { if (match('=')) { // >= return TOKEN_GREATER_EQUAL; } if (match('>') && match('=')) { return TOKEN_RIGHT_SHIFT_EQUAL; } return TOKEN_RIGHT_ANGLE; // > } case '&': return match('&') ? TOKEN_AND_AND : TOKEN_AND; case '|': return match('|') ? TOKEN_OR_OR : TOKEN_OR; case '~': return TOKEN_TILDE; case '^': return match('=') ? TOKEN_XOR_EQUAL : TOKEN_XOR; default: return token_type_t::TOKEN_NOT_IN_THIS_TYPE; } } inline std::string string_advance() { module.s_cursor.guard++; char escape_char = '\\'; std::stringstream buf; while (module.s_cursor.source[module.s_cursor.guard] != '\"' && !at_eof()) { char guard = module.s_cursor.source[module.s_cursor.guard]; if (guard == '\n') { std::cerr << "[ERROR] line: " << module.s_cursor.line << ". column: " << module.s_cursor.column <<". string cannot newline."; } // 处理转义字符 if (guard == escape_char) { // 跳过转义字符第一个 module.s_cursor.guard++; guard = module.s_cursor.source[module.s_cursor.guard]; switch (guard) { case 'n': guard = '\n'; break; case 't': guard = '\t'; break; case 'r': guard = '\r'; break; case 'b': guard = '\b'; break; case 'f': guard = '\f'; break; case 'a': guard = '\a'; break; case 'v': guard = '\v'; break; case '0': guard = '\0'; break; case '\\': case '\'': case '\"': break; default: std::cerr << "[ERROR] line: " << module.s_cursor.line << ". column: " << module.s_cursor.column <<". unknown escape char " << guard; } } buf << guard; guard_advance(); } //跳过close char module.s_cursor.guard++; return buf.str(); } inline long number_convert(std::string word, int base) { try { long decimal = std::stol(word, 0, base); return decimal; } catch (const std::invalid_argument& e) { std::cerr << "[ERROR] line: " << module.s_cursor.line << ". column: " << module.s_cursor.column <<". Invalid number: " << word << std::endl; return 0; } catch (const std::out_of_range& e) { std::cerr << "[ERROR] line: " << module.s_cursor.line << ". column: " << module.s_cursor.column <<". Number out of range: " << word << std::endl; return 0; } } inline std::string hex_number_advance() { module.s_cursor.guard += 2; // 跳过 0x while (is_hex_number(module.s_cursor.source[module.s_cursor.guard]) && !at_eof()) { guard_advance(); } return gen_word(); } inline std::string oct_number_advance() { module.s_cursor.guard += 2; // 跳过 0o while (is_oct_number(module.s_cursor.source[module.s_cursor.guard]) && !at_eof()) { guard_advance(); } return gen_word(); } inline std::string bin_number_advance() { module.s_cursor.guard += 2; // 跳过 0b while (is_bin_number(module.s_cursor.source[module.s_cursor.guard]) && !at_eof()) { guard_advance(); } return gen_word(); } inline std::string number_advance() { while(is_number(module.s_cursor.source[module.s_cursor.guard]) && !at_eof()) { guard_advance(); } // 处理小数部分 if (module.s_cursor.source[module.s_cursor.guard] == '.' && is_number(module.s_cursor.source[module.s_cursor.guard + 1])) { guard_advance(); // 跳过小数点 while(is_number(module.s_cursor.source[module.s_cursor.guard]) && !at_eof()) { guard_advance(); } } // 处理科学计数法 if ((module.s_cursor.source[module.s_cursor.guard] == 'e' || module.s_cursor.source[module.s_cursor.guard] == 'E') && (is_number(module.s_cursor.source[module.s_cursor.guard + 1]) || module.s_cursor.source[module.s_cursor.guard + 1] == '+' || module.s_cursor.source[module.s_cursor.guard + 1] == '-')) { guard_advance(); // 跳过 e 或 E if (module.s_cursor.source[module.s_cursor.guard] == '+' || module.s_cursor.source[module.s_cursor.guard] == '-') { guard_advance(); // 跳过符号 } while(is_number(module.s_cursor.source[module.s_cursor.guard]) && !at_eof()) { guard_advance(); } } return gen_word(); } static token_type_t scanner_ident(std::string word, int length) { switch (word[0]) { case 'a': { switch (word[1]) { case 'r': { if (word.substr(2, 3) == "ray") { return TOKEN_ARR; } } } break; } case 'b': switch (word[1]) { case 'o': if (word.substr(2, 2) == "ol") { return TOKEN_BOOL; case 'r': if (word.substr(2, 3) == "eak") { return TOKEN_BREAK; } } break; case 'c': switch (word[1]) { case 'o': if (word.substr(2, 6) == "ntinue") { return TOKEN_CONTINUE; } } break; case 'e': if (word.substr(1, 3) == "lse") { if (length == 3 && word[3] == 'i') { return TOKEN_ELSE_IF; } return TOKEN_ELSE; } return scanner_rest(word, length, 1, 3, "lse", TOKEN_ELSE); case 'f': { switch (word[1]) { case 'n': return scanner_rest(word, length, 2, 0, "", TOKEN_FN); case 'a': return scanner_rest(word, length, 2, 3, "lse", TOKEN_FALSE); case 'l': return scanner_rest(word, length, 2, 3, "oat", TOKEN_FLOAT); case '3': return scanner_rest(word, length, 2, 1, "2", TOKEN_F32); case '6': return scanner_rest(word, length, 2, 1, "4", TOKEN_F64); case 'o': return scanner_rest(word, length, 2, 1, "r", TOKEN_FOR); } break; } case 'g': return scanner_rest(word, length, 1, 1, "o", TOKEN_GO); case 'i': { if (length == 2 && word[1] == 'n') { return TOKEN_IN; } else if (length == 2 && word[1] == 's') { return TOKEN_IS; } else if (length == 3 && word[1] == 'n' && word[2] == 't') { return TOKEN_INT; } switch (word[1]) { case 'm': return scanner_rest(word, length, 2, 4, "port", TOKEN_IMPORT); case 'f': return scanner_rest(word, length, 2, 0, "", TOKEN_IF); case 'n': return scanner_rest(word, length, 2, 7, "terface", TOKEN_INTERFACE); case '8': return scanner_rest(word, length, 2, 0, "", TOKEN_I8); case '1': return scanner_rest(word, length, 2, 1, "6", TOKEN_I16); case '3': return scanner_rest(word, length, 2, 1, "2", TOKEN_I32); case '6': return scanner_rest(word, length, 2, 1, "4", TOKEN_I64); } break; } case 'l': { return scanner_rest(word, length, 1, 2, "et", TOKEN_LET); } case 'n': switch (word[1]) { case 'u': // null return scanner_rest(word, length, 2, 2, "ll", TOKEN_NULL); // case 'e':// new, new 识别成 ident 在 parser 采用固定语法结构时才会被识别成 new // return scanner_rest(word, length, 2, 1, "w", TOKEN_NEW); } break; case 'p': return scanner_rest(word, length, 1, 2, "tr", TOKEN_PTR); case 's': { // self,string,struct,sizeof,sett switch (word[1]) { case 'e': { switch (word[2]) { case 't': return scanner_rest(word, length, 3, 0, "", TOKEN_SET); case 'l': // select return scanner_rest(word, length, 3, 3, "ect", TOKEN_SELECT); } } } if (length == 6 && word[1] == 't' && word[2] == 'r') { switch (word[3]) { case 'i': return scanner_rest(word, length, 4, 2, "ng", TOKEN_STRING); case 'u': return scanner_rest(word, length, 4, 2, "ct", TOKEN_STRUCT); } } break; } case 't': { // tup/throw/type/true switch (word[1]) { case 'h': return scanner_rest(word, length, 2, 3, "row", TOKEN_THROW); case 'y': // type return scanner_rest(word, length, 2, 2, "pe", TOKEN_TYPE); case 'u': // tup return scanner_rest(word, length, 2, 1, "p", TOKEN_TUP); case 'r': { switch (word[2]) { case 'y': return scanner_rest(word, length, 3, 0, "", TOKEN_TRY); case 'u': return scanner_rest(word, length, 3, 1, "e", TOKEN_TRUE); } break; } } break; } case 'v': { switch (word[1]) { case 'a': return scanner_rest(word, length, 2, 1, "r", TOKEN_VAR); case 'e': // vec return scanner_rest(word, length, 2, 1, "c", TOKEN_VEC); case 'o': // void return scanner_rest(word, length, 2, 2, "id", TOKEN_VOID); } } case 'u': { switch (word[1]) { case 'i': return scanner_rest(word, length, 2, 2, "nt", TOKEN_UINT); case '8': return scanner_rest(word, length, 2, 0, "", TOKEN_U8); case '1': return scanner_rest(word, length, 2, 1, "6", TOKEN_U16); case '3': return scanner_rest(word, length, 2, 1, "2", TOKEN_U32); case '6': return scanner_rest(word, length, 2, 1, "4", TOKEN_U64); } break; } case 'm': { // map switch (word[1]) { case 'a': { switch (word[2]) { case 'p': return scanner_rest(word, length, 3, 0, "", TOKEN_MAP); case 't': return scanner_rest(word, length, 3, 2, "ch", TOKEN_MATCH); } } } } case 'r': { return scanner_rest(word, length, 1, 5, "eturn", TOKEN_RETURN); } } return TOKEN_IDENT; } inline token_t item() { module.s_cursor.length = 0; // 重置长度 module.s_cursor.guard = module.s_cursor.current; // 重置游标位置 if (is_alpha(module.s_cursor.source[module.s_cursor.guard])) { std::string word = ident_advance(); return token_t(ident) } } };