ncore-ui-kit/src/components/codeViewer/parser.ts

import type {
    TokenType,
    Token
} from "./type";

const keywordsByLanguage: Record<string, string[]> = {
    "csharp": [
        "protected",
        "namespace",
        "interface",
        "internal",
        "readonly",
        "override",
        "abstract",
        "continue",
        "private",
        "virtual",
        "decimal",
        "finally",
        "public",
        "static",
        "sealed",
        "double",
        "ushort",
        "string",
        "object",
        "switch",
        "return",
        "typeof",
        "using",
        "float",
        "sbyte",
        "short",
        "ulong",
        "class",
        "while",
        "break",
        "catch",
        "throw",
        "false",
        "bool",
        "byte",
        "char",
        "long",
        "uint",
        "this",
        "base",
        "else",
        "case",
        "null",
        "true",
        "void",
        "get",
        "set",
        "int",
        "new",
        "for",
        "try",
        "if",
        "do"
    ],
    "typescript": [
        "implements",
        "instanceof",
        "interface",
        "undefined",
        "namespace",
        "continue",
        "function",
        "extends",
        "default",
        "finally",
        "unknown",
        "boolean",
        "switch",
        "return",
        "import",
        "export",
        "typeof",
        "number",
        "string",
        "symbol",
        "object",
        "while",
        "break",
        "const",
        "class",
        "async",
        "await",
        "catch",
        "throw",
        "super",
        "false",
        "never",
        "else",
        "case",
        "type",
        "from",
        "this",
        "void",
        "null",
        "true",
        "enum",
        "for",
        "let",
        "var",
        "try",
        "new",
        "any",
        "if",
        "do"
    ],
    "tsx": [
        "implements",
        "instanceof",
        "interface",
        "undefined",
        "namespace",
        "continue",
        "function",
        "extends",
        "default",
        "finally",
        "unknown",
        "boolean",
        "switch",
        "return",
        "import",
        "export",
        "typeof",
        "number",
        "string",
        "symbol",
        "object",
        "while",
        "break",
        "const",
        "class",
        "async",
        "await",
        "catch",
        "throw",
        "super",
        "false",
        "never",
        "else",
        "case",
        "type",
        "from",
        "this",
        "void",
        "null",
        "true",
        "enum",
        "for",
        "let",
        "var",
        "try",
        "new",
        "any",
        "if",
        "do"
    ],
    "javascript": [
        "implements",
        "instanceof",
        "interface",
        "undefined",
        "continue",
        "function",
        "extends",
        "default",
        "finally",
        "switch",
        "return",
        "import",
        "export",
        "typeof",
        "while",
        "break",
        "const",
        "class",
        "async",
        "await",
        "catch",
        "throw",
        "super",
        "false",
        "else",
        "case",
        "type",
        "from",
        "this",
        "void",
        "null",
        "true",
        "for",
        "let",
        "var",
        "try",
        "new",
        "if",
        "do"
    ],
    "jsx": [
        "implements",
        "instanceof",
        "interface",
        "undefined",
        "continue",
        "function",
        "extends",
        "default",
        "finally",
        "switch",
        "return",
        "import",
        "export",
        "typeof",
        "while",
        "break",
        "const",
        "class",
        "async",
        "await",
        "catch",
        "throw",
        "super",
        "false",
        "else",
        "case",
        "type",
        "from",
        "this",
        "void",
        "null",
        "true",
        "for",
        "let",
        "var",
        "try",
        "new",
        "if",
        "do"
    ],
    "sql": [
        "REFERENCES",
        "PRIMARY",
        "FOREIGN",
        "BETWEEN",
        "SELECT",
        "INSERT",
        "VALUES",
        "UPDATE",
        "DELETE",
        "CREATE",
        "HAVING",
        "OFFSET",
        "EXISTS",
        "WHERE",
        "TABLE",
        "ALTER",
        "INDEX",
        "INNER",
        "RIGHT",
        "OUTER",
        "GROUP",
        "ORDER",
        "LIMIT",
        "FROM",
        "INTO",
        "DROP",
        "JOIN",
        "LEFT",
        "FULL",
        "NULL",
        "LIKE",
        "SET",
        "KEY",
        "AND",
        "NOT",
        "ON",
        "BY",
        "AS",
        "OR",
        "IS",
        "IN"
    ],
    "bash": [
        "install",
        "remove",
        "docker",
        "export",
        "source",
        "return",
        "start",
        "build",
        "while",
        "yarn",
        "pnpm",
        "test",
        "lint",
        "echo",
        "sudo",
        "brew",
        "bash",
        "then",
        "done",
        "esac",
        "else",
        "elif",
        "case",
        "npm",
        "npx",
        "bun",
        "add",
        "run",
        "dev",
        "apt",
        "git",
        "zsh",
        "for",
        "cd",
        "ls",
        "sh",
        "fi",
        "if",
        "do",
        "in"
    ],
    "python": [
        "nonlocal",
        "continue",
        "finally",
        "lambda",
        "global",
        "import",
        "except",
        "return",
        "assert",
        "False",
        "yield",
        "class",
        "raise",
        "while",
        "break",
        "async",
        "await",
        "elif",
        "pass",
        "None",
        "True",
        "with",
        "from",
        "else",
        "def",
        "and",
        "not",
        "del",
        "try",
        "for",
        "or",
        "is",
        "in",
        "as",
        "if"
    ]
};

const allKeywordsList = [
    "implements",
    "instanceof",
    "REFERENCES",
    "protected",
    "namespace",
    "interface",
    "undefined",
    "internal",
    "readonly",
    "override",
    "abstract",
    "continue",
    "function",
    "nonlocal",
    "private",
    "virtual",
    "decimal",
    "finally",
    "extends",
    "default",
    "unknown",
    "boolean",
    "PRIMARY",
    "FOREIGN",
    "BETWEEN",
    "install",
    "public",
    "static",
    "sealed",
    "double",
    "ushort",
    "string",
    "object",
    "switch",
    "return",
    "typeof",
    "import",
    "export",
    "number",
    "symbol",
    "SELECT",
    "INSERT",
    "VALUES",
    "UPDATE",
    "DELETE",
    "CREATE",
    "HAVING",
    "OFFSET",
    "EXISTS",
    "remove",
    "docker",
    "source",
    "lambda",
    "global",
    "except",
    "assert",
    "using",
    "float",
    "sbyte",
    "short",
    "ulong",
    "class",
    "while",
    "break",
    "catch",
    "throw",
    "false",
    "const",
    "async",
    "await",
    "super",
    "never",
    "WHERE",
    "TABLE",
    "ALTER",
    "INDEX",
    "INNER",
    "RIGHT",
    "OUTER",
    "GROUP",
    "ORDER",
    "LIMIT",
    "start",
    "build",
    "False",
    "yield",
    "raise",
    "bool",
    "byte",
    "char",
    "long",
    "uint",
    "this",
    "base",
    "else",
    "case",
    "null",
    "true",
    "void",
    "type",
    "from",
    "enum",
    "FROM",
    "INTO",
    "DROP",
    "JOIN",
    "LEFT",
    "FULL",
    "NULL",
    "LIKE",
    "yarn",
    "pnpm",
    "test",
    "lint",
    "echo",
    "sudo",
    "brew",
    "bash",
    "then",
    "done",
    "esac",
    "elif",
    "pass",
    "None",
    "True",
    "with",
    "get",
    "set",
    "int",
    "new",
    "for",
    "try",
    "let",
    "var",
    "any",
    "SET",
    "KEY",
    "AND",
    "NOT",
    "npm",
    "npx",
    "bun",
    "add",
    "run",
    "dev",
    "apt",
    "git",
    "zsh",
    "def",
    "and",
    "not",
    "del",
    "if",
    "do",
    "ON",
    "BY",
    "AS",
    "OR",
    "IS",
    "IN",
    "cd",
    "ls",
    "sh",
    "fi",
    "in",
    "or",
    "is",
    "as"
];

const tokenRegexCache: Record<string, RegExp> = {};

const getTokenRegex = (language?: string): RegExp => {
    const cacheKey = language || "default";
    const cachedRegex = tokenRegexCache[cacheKey];
    if (cachedRegex) {
        return cachedRegex;
    }

    let keywordList = allKeywordsList;
    if (language) {
        const langKeywords = keywordsByLanguage[language.toLowerCase()];
        if (langKeywords) {
            keywordList = langKeywords;
        }
    }

    const keywordsRegexStr = "(?<!\\.)\\b(?:" + keywordList.join("|") + ")\\b";

    const keywordControlRegexStr = "(?<!\\.)\\b(?:return|if|else|for|while|try|catch|finally|switch|case|break|continue|throw|yield)\\b";

    let commentRegexStr = "//.*|/\\*[\\s\\S]*?\\*/|#.*|<!--[\\s\\S]*?-->";
    if (language) {
        switch (language.toLowerCase()) {
            case "bash":
            case "python":
                commentRegexStr = "#.*";
                break;
            case "sql":
                commentRegexStr = "--.*|/\\*[\\s\\S]*?\\*/";
                break;
            case "html":
            case "xml":
                commentRegexStr = "<!--[\\s\\S]*?-->";
                break;
            case "javascript":
            case "typescript":
            case "jsx":
            case "tsx":
            case "csharp":
                commentRegexStr = "//.*|/\\*[\\s\\S]*?\\*/";
                break;
        }
    }

    let bashTextRegexStr = "";
    if (language && language.toLowerCase() === "bash") {
        bashTextRegexStr = "(?<text>(?:[a-zA-Z0-9_@]+[.:/-]+)+[a-zA-Z0-9_@]+)|";
    }

    const regexStr = "(?<comment>" + commentRegexStr + ")|" +
        '(?<string>(["\\\'`])(?:(?=(\\\\?))\\4.)*?\\3)|' +
        bashTextRegexStr +
        "(?<keywordControl>" + keywordControlRegexStr + ")|" +
        "(?<keyword>" + keywordsRegexStr + ")|" +
        "(?<htmlTag>(?<=<\\/?)[a-z][a-zA-Z0-9-]*\\b)|" +
        "(?<component>\\b[A-Z][a-zA-Z0-9_$]*\\b)|" +
        "(?<function>\\b[a-zA-Z_$][a-zA-Z0-9_$]*(?=\\s*\\()|\\b[a-zA-Z_$][a-zA-Z0-9_$]*(?=\\s*[:=]\\s*(?:async\\s+)?(?:\\([^)]*\\)|[a-zA-Z_$][a-zA-Z0-9_$]*)\\s*=>)|\\b[a-zA-Z_$][a-zA-Z0-9_$]*(?=\\s*[:=]\\s*function\\b))|" +
        "(?<property>\\b[a-zA-Z_$][a-zA-Z0-9_$]*(?=\\s*[:=]))|" +
        "(?<variable>\\b[a-zA-Z_$][a-zA-Z0-9_$]*\\b)|" +
        "(?<number>\\b\\d+(?:\\.\\d+)?\\b)|" +
        "(?<arrow>=>)|" +
        "(?<punctuationTag>[<>/])|" +
        "(?<operator>[+\\-*=!&|%^~]+)|" +
        "(?<punctuation>[{}()\\[\\].,;:])";

    const regex = new RegExp(regexStr, "g");
    tokenRegexCache[cacheKey] = regex;

    return regex;
};

export const parseCode = (code: string, language?: string): Token[] => {
    const tokens: Token[] = [];
    let lastIndex = 0;

    const tokenRegex = getTokenRegex(language);
    tokenRegex.lastIndex = 0;

    let match: RegExpExecArray | null;

    while ((match = tokenRegex.exec(code)) !== null) {
        if (match.index > lastIndex) {
            tokens.push({
                type: "text",
                value: code.substring(lastIndex, match.index)
            });
        }

        const groups = match.groups || {};
        const type = Object.keys(groups).find(key => groups[key] !== undefined) as TokenType;

        if (type) {
            tokens.push({
                type,
                value: groups[type] as string
            });
        }

        lastIndex = match.index + match[0].length;
    }

    if (lastIndex < code.length) {
        tokens.push({
            type: "text",
            value: code.substring(lastIndex)
        });
    }

    return tokens;
};

export const splitTokensIntoLines = (tokens: Token[]): Token[][] => {
    const lines: Token[][] = [];
    let currentLine: Token[] = [];

    tokens.forEach((token) => {
        if (token.value.includes("\n")) {
            const parts = token.value.split("\n");
            parts.forEach((part, index) => {
                if (part.length > 0) {
                    currentLine.push({
                        ...token,
                        value: part
                    });
                }

                if (index < parts.length - 1) {
                    lines.push(currentLine);

                    currentLine = [];
                }
            });
        } else {
            currentLine.push(token);
        }
    });

    lines.push(currentLine);

    return lines;
};

export const getLineIndentation = (lineTokens: Token[]): number => {
    let indentation = 0;
    for (const token of lineTokens) {
        if (token.type === "text") {
            const match = token.value.match(/^(\s+)/);

            if(match) {
                const matchLen = (match[1] as string).length;

                indentation += matchLen;

                if(matchLen !== token.value.length) {
                    break;
                }
            } else if (token.value.length > 0) {
                break;
            }
        } else {
            break;
        }
    }
    return indentation;
};