add syntax highlighting (WIP)

res/layouts/console.xml

@@ -32,10 +32,9 @@
             autoresize='true'
             margin='0'
             padding='5'
-            editable='false'
             multiline='true'
             line-numbers='true'
-            text-color="#FFFFFFA0"
+            syntax='lua'
             size-func="gui.get_viewport()[1]-350,40"
             gravity="top-left"
             text-wrap='false'

src/coders/commons.cpp

@@ -214,10 +214,14 @@ std::string_view BasicParser::readUntil(char c) {
     return source.substr(start, pos - start);
 }
 
-std::string_view BasicParser::readUntil(std::string_view s) {
+std::string_view BasicParser::readUntil(std::string_view s, bool nothrow) {
     int start = pos;
     size_t found = source.find(s, pos);
     if (found == std::string::npos) {
+        if (nothrow) {
+            pos = source.size();
+            return source.substr(start);
+        }
         throw error(util::quote(std::string(s))+" expected");
     }
     skip(found - pos);

src/coders/commons.hpp

@@ -105,7 +105,7 @@ protected:
     parsing_error error(const std::string& message);
 public:
     std::string_view readUntil(char c);
-    std::string_view readUntil(std::string_view s);
+    std::string_view readUntil(std::string_view s, bool nothrow);
     std::string_view readUntilWhitespace();
     std::string_view readUntilEOL();
     std::string parseName();

src/coders/lua_parsing.cpp

@@ -119,24 +119,28 @@ public:
                 );
                 continue;
             } else if (is_digit(c)) {
-                auto value = parseNumber(1);
+                dv::value value;
+                auto tag = TokenTag::UNEXPECTED;
+                try {
+                    value = parseNumber(1);
+                    tag = value.isInteger() ? TokenTag::INTEGER
+                                            : TokenTag::NUMBER;
+                } catch (const parsing_error& err) {}
+
                 auto literal = source.substr(start.pos, pos - start.pos);
-                emitToken(
+                emitToken(tag, std::string(literal), start);
-                    value.isInteger() ? TokenTag::INTEGER : TokenTag::NUMBER,
-                    std::string(literal),
-                    start
-                );
                 continue;
             }
             switch (c) {
                 case '(': case '[': case '{': 
                     if (isNext("[==[")) {
-                        readUntil("]==]");
+                        auto string = readUntil("]==]", true);
                         skip(4);
+                        emitToken(TokenTag::COMMENT, std::string(string)+"]==]", start);
                         continue;
                     } else if (isNext("[[")) {
                         skip(2);
-                        auto string = readUntil("]]");
+                        auto string = readUntil("]]", true);
                         skip(2);
                         emitToken(TokenTag::STRING, std::string(string), start);
                         continue;
@@ -154,7 +158,7 @@ public:
                     continue;
                 case '\'': case '"': {
                     skip(1);
-                    auto string = parseString(c);
+                    auto string = parseString(c, false);
                     emitToken(TokenTag::STRING, std::move(string), start);
                     continue;
                 }
@@ -163,6 +167,8 @@ public:
             if (is_lua_operator_start(c)) {
                 auto text = parseOperator();
                 if (text == "--") {
+                    auto string = readUntilEOL();
+                    emitToken(TokenTag::COMMENT, std::string(string), start);
                     skipLine();
                     continue;
                 }

src/coders/lua_parsing.hpp

@@ -12,7 +12,7 @@ namespace lua {
 
     enum class TokenTag {
         KEYWORD, NAME, INTEGER, NUMBER, OPEN_BRACKET, CLOSE_BRACKET, STRING,
-        OPERATOR, COMMA, SEMICOLON, UNEXPECTED
+        OPERATOR, COMMA, SEMICOLON, UNEXPECTED, COMMENT
     };
 
     struct Token {

src/devtools/syntax_highlighting.cpp

@@ -0,0 +1,72 @@
+#include "syntax_highlighting.hpp"
+
+#include "coders/commons.hpp"
+#include "coders/lua_parsing.hpp"
+#include "graphics/core/Font.hpp"
+
+using namespace devtools;
+
+static std::unique_ptr<FontStylesScheme> build_styles(
+    const std::vector<lua::Token>& tokens
+) {
+    FontStylesScheme styles {
+        {
+            {false, false, glm::vec4(0.8f, 0.8f, 0.8f, 1)}, // default
+            {true, false, glm::vec4(0.9, 0.6f, 0.4f, 1)},   // keyword
+            {false, false, glm::vec4(0.4, 0.8f, 0.5f, 1)},  // string
+            {false, false, glm::vec4(0.3, 0.3f, 0.3f, 1)},  // comment
+            {false, false, glm::vec4(0.4, 0.45f, 0.5f, 1)}, // self
+            {true, false, glm::vec4(1.0f, 0.2f, 0.1f, 1)}, // unexpected
+        }, 
+        {}
+    };
+    size_t offset = 0;
+    for (int i = 0; i < tokens.size(); i++) {
+        const auto& token = tokens.at(i);
+        if (token.tag != lua::TokenTag::KEYWORD &&
+            token.tag != lua::TokenTag::STRING &&
+            token.tag != lua::TokenTag::INTEGER &&
+            token.tag != lua::TokenTag::NUMBER &&
+            token.tag != lua::TokenTag::COMMENT &&
+            token.tag != lua::TokenTag::UNEXPECTED) {
+            continue;
+        }
+        if (token.start.pos > offset) {
+            int n = token.start.pos - offset;
+            styles.map.insert(styles.map.end(), token.start.pos - offset, 0);
+        }
+        offset = token.end.pos;
+        int styleIndex;
+        switch (token.tag) {
+            case lua::TokenTag::KEYWORD: styleIndex = 1; break;
+            case lua::TokenTag::STRING:
+            case lua::TokenTag::INTEGER:
+            case lua::TokenTag::NUMBER: styleIndex = 2; break;
+            case lua::TokenTag::COMMENT: styleIndex = 3; break;
+            case lua::TokenTag::UNEXPECTED: styleIndex = 5; break;
+            default:
+                styleIndex = 0;
+                break;
+        }
+        styles.map.insert(
+            styles.map.end(), token.end.pos - token.start.pos, styleIndex
+        );
+    }
+    styles.map.push_back(0);
+    return std::make_unique<FontStylesScheme>(std::move(styles));
+}
+
+std::unique_ptr<FontStylesScheme> devtools::syntax_highlight(
+    const std::string& lang, std::string_view source
+) {
+    try {
+        if (lang == "lua") {
+            auto tokens = lua::tokenize("<string>", source);
+            return build_styles(tokens);
+        } else {
+            return nullptr;
+        }
+    } catch (const parsing_error& err) {
+        return nullptr;
+    }
+}

src/devtools/syntax_highlighting.hpp

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <string>
+#include <memory>
+
+struct FontStylesScheme;
+
+namespace devtools {
+    std::unique_ptr<FontStylesScheme> syntax_highlight(
+        const std::string& lang, std::string_view source
+    );
+}

src/graphics/core/Font.cpp

@@ -62,7 +62,7 @@ static inline void draw_glyph(
             pos.y + offset.y * right.y,
             right.x / glyphInterval,
             up.y,
-            -0.2f * style.italic,
+            -0.15f * style.italic,
             16,
             c,
             batch.getColor() * style.color
@@ -102,11 +102,11 @@ static inline void draw_text(
     const glm::vec3& right,
     const glm::vec3& up,
     float glyphInterval,
-    const FontStylesScheme* styles
+    const FontStylesScheme* styles,
+    size_t styleMapOffset
 ) {
-    static FontStylesScheme defStyles {
+    static FontStylesScheme defStyles {{{}}, {0}};
-        {{std::numeric_limits<size_t>::max()}},
+
-    };
     if (styles == nullptr) {
         styles = &defStyles;
     }
@@ -117,17 +117,12 @@ static inline void draw_text(
     int y = 0;
 
     do {
-        size_t entryIndex = 0;
+        for (size_t i = 0; i < text.length(); i++) {
-        int styleCharsCounter = -1;
+            uint c = text[i];
-        const FontStyle* style = &styles->palette.at(entryIndex);
+            size_t styleIndex = styles->map.at(
-
+                std::min(styles->map.size() - 1, i + styleMapOffset)
-        for (uint c : text) {
+            );
-            styleCharsCounter++;
+            const FontStyle& style = styles->palette.at(styleIndex);
-            if (styleCharsCounter > style->n && 
-                    entryIndex + 1 < styles->palette.size()) {
-                style = &styles->palette.at(++entryIndex);
-                styleCharsCounter = -1;
-            }
             if (!font.isPrintableChar(c)) {
                 x++;
                 continue;
@@ -143,7 +138,7 @@ static inline void draw_text(
                     right,
                     up,
                     glyphInterval,
-                    *style
+                    style
                 );
             }
             else if (charpage > page && charpage < next){
@@ -174,6 +169,7 @@ void Font::draw(
     int x,
     int y,
     const FontStylesScheme* styles,
+    size_t styleMapOffset,
     float scale
 ) const {
     draw_text(
@@ -182,7 +178,8 @@ void Font::draw(
         glm::vec3(glyphInterval*scale, 0, 0),
         glm::vec3(0, lineHeight*scale, 0),
         glyphInterval/static_cast<float>(lineHeight),
-        styles
+        styles,
+        styleMapOffset
     );
 }
 
@@ -190,6 +187,7 @@ void Font::draw(
     Batch3D& batch,
     std::wstring_view text,
     const FontStylesScheme* styles,
+    size_t styleMapOffset,
     const glm::vec3& pos,
     const glm::vec3& right,
     const glm::vec3& up
@@ -199,6 +197,7 @@ void Font::draw(
         right * static_cast<float>(glyphInterval),
         up * static_cast<float>(lineHeight),
         glyphInterval/static_cast<float>(lineHeight),
-        styles
+        styles,
+        styleMapOffset
     );
 }

src/graphics/core/Font.hpp

@@ -12,7 +12,6 @@ class Batch3D;
 class Camera;
 
 struct FontStyle {
-    size_t n = -1;
     bool bold = false;
     bool italic = false;
     glm::vec4 color {1, 1, 1, 1};
@@ -20,6 +19,7 @@ struct FontStyle {
 
 struct FontStylesScheme {
     std::vector<FontStyle> palette;
+    std::vector<ubyte> map;
 };
 
 class Font {
@@ -57,6 +57,7 @@ public:
         int x,
         int y,
         const FontStylesScheme* styles,
+        size_t styleMapOffset,
         float scale = 1
     ) const;
 
@@ -64,6 +65,7 @@ public:
         Batch3D& batch,
         std::wstring_view text,
         const FontStylesScheme* styles,
+        size_t styleMapOffset,
         const glm::vec3& pos,
         const glm::vec3& right={1, 0, 0},
         const glm::vec3& up={0, 1, 0}

src/graphics/render/TextsRenderer.cpp

@@ -98,13 +98,13 @@ void TextsRenderer::renderNote(
                                      pos + xvec * (width * 0.5f * preset.scale))) {
         return;
     }
-    static FontStylesScheme styles {};
     auto color = preset.color;
     batch.setColor(glm::vec4(color.r, color.g, color.b, color.a * opacity));
     font.draw(
         batch,
         text,
-        &styles,
+        nullptr,
+        0,
         pos - xvec * (width * 0.5f) * preset.scale,
         xvec * preset.scale,
         yvec * preset.scale

src/graphics/ui/elements/InventoryView.cpp

@@ -194,9 +194,9 @@ void SlotView::draw(const DrawContext* pctx, Assets* assets) {
         int y = pos.y+slotSize-16;
 
         batch->setColor({0, 0, 0, 1.0f});
-        font->draw(*batch, text, x+1, y+1, nullptr);
+        font->draw(*batch, text, x+1, y+1, nullptr, 0);
         batch->setColor(glm::vec4(1.0f));
-        font->draw(*batch, text, x, y, nullptr);
+        font->draw(*batch, text, x, y, nullptr, 0);
     }
 }
 

src/graphics/ui/elements/Label.cpp

@@ -203,10 +203,10 @@ void Label::draw(const DrawContext* pctx, Assets* assets) {
             if (i < cache.lines.size()-1) {
                 view = std::wstring_view(text.c_str()+offset, cache.lines.at(i+1).offset-offset);
             }
-            font->draw(*batch, view, pos.x, pos.y + i * totalLineHeight, styles.get());
+            font->draw(*batch, view, pos.x, pos.y + i * totalLineHeight, styles.get(), offset);
         }
     } else {
-        font->draw(*batch, text, pos.x, pos.y, styles.get());
+        font->draw(*batch, text, pos.x, pos.y, styles.get(), 0);
     }
 }
 

src/graphics/ui/elements/Plotter.cpp

@@ -52,7 +52,8 @@ void Plotter::draw(const DrawContext* pctx, Assets* assets) {
             string,
             pos.x + dmwidth + 2,
             pos.y + dmheight - y - labelsInterval,
-            nullptr
+            nullptr,
+            0
         );
     }
 }

src/graphics/ui/elements/TextBox.cpp

@@ -5,6 +5,7 @@
 #include <algorithm>
 
 #include "Label.hpp"
+#include "devtools/syntax_highlighting.hpp"
 #include "graphics/core/DrawContext.hpp"
 #include "graphics/core/Batch2D.hpp"
 #include "graphics/core/Font.hpp"
@@ -65,11 +66,10 @@ void TextBox::draw(const DrawContext* pctx, Assets* assets) {
     lcoord.y -= 2;
     auto batch = pctx->getBatch2D();
     batch->texture(nullptr);
+    batch->setColor(glm::vec4(1.0f));
     if (editable && int((Window::time() - caretLastMove) * 2) % 2 == 0) {
         uint line = label->getLineByTextIndex(caret);
         uint lcaret = caret - label->getTextLineOffset(line);
-        batch->setColor(glm::vec4(1.0f));
-
         int width = font->calcWidth(input, lcaret);
         batch->rect(lcoord.x + width, lcoord.y+label->getLineYOffset(line), 2, lineHeight);
     }
@@ -529,10 +529,21 @@ void TextBox::stepDefaultUp(bool shiftPressed, bool breakSelection) {
     }
 }
 
+void TextBox::refreshSyntax() {
+    if (!syntax.empty()) {
+        if (auto styles = devtools::syntax_highlight(
+                syntax, util::wstr2str_utf8(input)
+            )) {
+            label->setStyles(std::move(styles));
+        }
+    }
+}
+
 void TextBox::onInput() {
     if (subconsumer) {
         subconsumer(input);
     }
+    refreshSyntax();
 }
 
 void TextBox::performEditingKeyboardEvents(keycode key) {
@@ -710,6 +721,7 @@ const std::wstring& TextBox::getText() const {
 void TextBox::setText(const std::wstring& value) {
     this->input = value;
     input.erase(std::remove(input.begin(), input.end(), '\r'), input.end());
+    refreshSyntax();
 }
 
 const std::wstring& TextBox::getPlaceholder() const {
@@ -789,3 +801,12 @@ void TextBox::setShowLineNumbers(bool flag) {
 bool TextBox::isShowLineNumbers() const {
     return showLineNumbers;
 }
+
+void TextBox::setSyntax(const std::string& lang) {
+    syntax = lang;
+    if (syntax.empty()) {
+        label->setStyles(nullptr);
+    } else {
+        refreshSyntax();
+    }
+}

src/graphics/ui/elements/TextBox.hpp

@@ -57,6 +57,8 @@ namespace gui {
         bool autoresize = false;
         bool showLineNumbers = false;
 
+        std::string syntax;
+
         void stepLeft(bool shiftPressed, bool breakSelection);
         void stepRight(bool shiftPressed, bool breakSelection);
         void stepDefaultDown(bool shiftPressed, bool breakSelection);
@@ -84,6 +86,8 @@ namespace gui {
         void refreshLabel();
 
         void onInput();
+
+        void refreshSyntax();
     public:
         TextBox(
             std::wstring placeholder, 
@@ -219,5 +223,7 @@ namespace gui {
         virtual std::shared_ptr<UINode> getAt(glm::vec2 pos, std::shared_ptr<UINode> self) override;
         virtual void setOnUpPressed(const runnable &callback);
         virtual void setOnDownPressed(const runnable &callback);
+
+        virtual void setSyntax(const std::string& lang);
     };
 }

src/graphics/ui/gui_xml.cpp

@@ -357,6 +357,9 @@ static std::shared_ptr<UINode> readTextBox(UiXmlReader& reader, const xml::xmlel
     }
     textbox->setText(text);
 
+    if (element->has("syntax")) {
+        textbox->setSyntax(element->attr("syntax").getText());
+    }
     if (element->has("multiline")) {
         textbox->setMultiline(element->attr("multiline").asBool());
     }