Merge pull request #369 from MihailRis/curl

Network

.github/workflows/appimage.yml

@@ -23,7 +23,8 @@ jobs:
       - name: install dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y build-essential libglfw3-dev libglfw3 libglew-dev libglm-dev libpng-dev libopenal-dev libluajit-5.1-dev libvorbis-dev cmake squashfs-tools
+          sudo apt-get install -y build-essential libglfw3-dev libglfw3 libglew-dev \
+            libglm-dev libpng-dev libopenal-dev libluajit-5.1-dev libvorbis-dev libcurl4-openssl-dev cmake squashfs-tools
           # fix luajit paths
           sudo ln -s /usr/lib/x86_64-linux-gnu/libluajit-5.1.a /usr/lib/x86_64-linux-gnu/liblua5.1.a
           sudo ln -s /usr/include/luajit-2.1 /usr/include/lua

.github/workflows/cmake.yml

@@ -39,7 +39,7 @@ jobs:
     #   make && make install INSTALL_INC=/usr/include/lua
       run: |
           sudo apt-get update
-          sudo apt-get install libglfw3-dev libglfw3 libglew-dev libglm-dev libpng-dev libopenal-dev libluajit-5.1-dev libvorbis-dev libgtest-dev
+          sudo apt-get install libglfw3-dev libglfw3 libglew-dev libglm-dev libpng-dev libopenal-dev libluajit-5.1-dev libvorbis-dev libgtest-dev libcurl4-openssl-dev
           # fix luajit paths
           sudo ln -s /usr/lib/x86_64-linux-gnu/libluajit-5.1.a /usr/lib/x86_64-linux-gnu/liblua-5.1.a
           sudo ln -s /usr/include/luajit-2.1 /usr/include/lua

res/scripts/classes.lua

@@ -34,3 +34,31 @@ cameras.get = function(name)
     wrappers[name] = wrapper
     return wrapper
 end
+
+
+local Socket = {__index={
+    send=function(self, ...) return network.__send(self.id, ...) end,
+    recv=function(self, ...) return network.__recv(self.id, ...) end,
+    close=function(self) return network.__close(self.id) end,
+    is_alive=function(self) return network.__is_alive(self.id) end,
+    is_connected=function(self) return network.__is_connected(self.id) end,
+}}
+
+network.tcp_connect = function(address, port, callback)
+    local socket = setmetatable({id=0}, Socket)
+    socket.id = network.__connect(address, port, function(id)
+        callback(socket)
+    end)
+    return socket
+end
+
+local ServerSocket = {__index={
+    close=function(self) return network.__closeserver(self.id) end,
+    is_open=function(self) return network.__is_serveropen(self.id) end,
+}}
+
+network.tcp_open = function(port, handler)
+    return setmetatable({id=network.__open(port, function(id)
+        handler(setmetatable({id=id}, Socket))
+    end)}, ServerSocket)
+end

src/CMakeLists.txt

@@ -15,6 +15,7 @@ find_package(GLEW REQUIRED)
 find_package(OpenAL REQUIRED)
 find_package(ZLIB REQUIRED)
 find_package(PNG REQUIRED)
+find_package(CURL REQUIRED)
 if (NOT APPLE)
     find_package(EnTT REQUIRED)
 endif()
@@ -61,5 +62,6 @@ if(UNIX)
 endif()
 
 include_directories(${LUA_INCLUDE_DIR})
+include_directories(${CURL_INCLUDE_DIR})
 target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
-target_link_libraries(${PROJECT_NAME} ${LIBS} glfw OpenGL::GL ${OPENAL_LIBRARY} GLEW::GLEW ZLIB::ZLIB PNG::PNG ${VORBISLIB} ${LUA_LIBRARIES} ${CMAKE_DL_LIBS})
+target_link_libraries(${PROJECT_NAME} ${LIBS} glfw OpenGL::GL ${OPENAL_LIBRARY} GLEW::GLEW ZLIB::ZLIB PNG::PNG CURL::libcurl ${VORBISLIB} ${LUA_LIBRARIES} ${CMAKE_DL_LIBS})

src/coders/json.cpp

@@ -73,7 +73,7 @@ void stringifyValue(
             break;
         }
         case value_type::string:
-            ss << util::escape(value.asString());
+            ss << util::escape(value.asString(), !nice);
             break;
         case value_type::number:
             ss << std::setprecision(15) << value.asNumber();
@@ -241,6 +241,8 @@ dv::value Parser::parseValue() {
             return INFINITY;
         } else if (literal == "nan") {
             return NAN;
+        } else if (literal == "null") {
+            return nullptr;
         }
         throw error("invalid keyword " + literal);
     }

src/engine.cpp

@@ -30,6 +30,7 @@
 #include "logic/EngineController.hpp"
 #include "logic/CommandsInterpreter.hpp"
 #include "logic/scripting/scripting.hpp"
+#include "network/Network.hpp"
 #include "util/listutil.hpp"
 #include "util/platform.hpp"
 #include "window/Camera.hpp"
@@ -72,7 +73,8 @@ static std::unique_ptr<ImageData> load_icon(const fs::path& resdir) {
 
 Engine::Engine(EngineSettings& settings, SettingsHandler& settingsHandler, EnginePaths* paths) 
     : settings(settings), settingsHandler(settingsHandler), paths(paths),
-      interpreter(std::make_unique<cmd::CommandsInterpreter>())
+      interpreter(std::make_unique<cmd::CommandsInterpreter>()),
+      network(network::Network::create(settings.network))
 {
     paths->prepare();
     loadSettings();
@@ -191,6 +193,7 @@ void Engine::mainloop() {
                 : settings.display.framerate.get()
         );
 
+        network->update();
         processPostRunnables();
 
         Window::swapBuffers();
@@ -235,6 +238,7 @@ Engine::~Engine() {
     gui.reset();
     logger.info() << "gui finished";
     audio::close();
+    network.reset();
     scripting::close();
     logger.info() << "scripting finished";
     Window::terminate();
@@ -485,3 +489,7 @@ void Engine::postRunnable(const runnable& callback) {
 SettingsHandler& Engine::getSettingsHandler() {
     return settingsHandler;
 }
+
+network::Network& Engine::getNetwork() {
+    return *network;
+}

src/engine.hpp

@@ -36,6 +36,10 @@ namespace cmd {
     class CommandsInterpreter;
 }
 
+namespace network {
+    class Network;
+}
+
 class initialize_error : public std::runtime_error {
 public:
     initialize_error(const std::string& message) : std::runtime_error(message) {}
@@ -55,6 +59,7 @@ class Engine : public util::ObjectsKeeper {
     std::recursive_mutex postRunnablesMutex;
     std::unique_ptr<EngineController> controller;
     std::unique_ptr<cmd::CommandsInterpreter> interpreter;
+    std::unique_ptr<network::Network> network;
     std::vector<std::string> basePacks;
 
     uint64_t frame = 0;
@@ -147,4 +152,6 @@ public:
     PacksManager createPacksManager(const fs::path& worldFolder);
 
     SettingsHandler& getSettingsHandler();
+
+    network::Network& getNetwork();
 };

src/logic/scripting/lua/libs/api_lua.hpp

@@ -32,6 +32,7 @@ extern const luaL_Reg inventorylib[];
 extern const luaL_Reg itemlib[];
 extern const luaL_Reg jsonlib[];
 extern const luaL_Reg mat4lib[];
+extern const luaL_Reg networklib[];
 extern const luaL_Reg packlib[];
 extern const luaL_Reg particleslib[]; // gfx.particles
 extern const luaL_Reg playerlib[];

src/logic/scripting/lua/libs/libfile.cpp

@@ -149,24 +149,24 @@ static int l_read_bytes(lua::State* L) {
     );
 }
 
-static int read_bytes_from_table(
+static void read_bytes_from_table(
     lua::State* L, int tableIndex, std::vector<ubyte>& bytes
 ) {
     if (!lua::istable(L, tableIndex)) {
         throw std::runtime_error("table expected");
     } else {
-        lua::pushnil(L);
+        size_t size = lua::objlen(L, tableIndex);
-        while (lua::next(L, tableIndex - 1) != 0) {
+        for (size_t i = 0; i < size; i++) {
+            lua::rawgeti(L, i + 1, tableIndex);
             const int byte = lua::tointeger(L, -1);
+            lua::pop(L);
             if (byte < 0 || byte > 255) {
                 throw std::runtime_error(
                     "invalid byte '" + std::to_string(byte) + "'"
                 );
             }
             bytes.push_back(byte);
-            lua::pop(L);
         }
-        return 1;
     }
 }
 
@@ -181,14 +181,10 @@ static int l_write_bytes(lua::State* L) {
     }
 
     std::vector<ubyte> bytes;
-    int result = read_bytes_from_table(L, -1, bytes);
+    read_bytes_from_table(L, 2, bytes);
-    if (result != 1) {
+    return lua::pushboolean(
-        return result;
+        L, files::write_bytes(path, bytes.data(), bytes.size())
-    } else {
+    );
-        return lua::pushboolean(
-            L, files::write_bytes(path, bytes.data(), bytes.size())
-        );
-    }
 }
 
 static int l_list_all_res(lua::State* L, const std::string& path) {
@@ -227,39 +223,29 @@ static int l_list(lua::State* L) {
 static int l_gzip_compress(lua::State* L) {
     std::vector<ubyte> bytes;
 
-    int result = read_bytes_from_table(L, -1, bytes);
+    read_bytes_from_table(L, 1, bytes);
+    auto compressed_bytes = gzip::compress(bytes.data(), bytes.size());
+    int newTable = lua::gettop(L);
 
-    if (result != 1) {
+    for (size_t i = 0; i < compressed_bytes.size(); i++) {
-        return result;
+        lua::pushinteger(L, compressed_bytes.data()[i]);
-    } else {
+        lua::rawseti(L, i + 1, newTable);
-        auto compressed_bytes = gzip::compress(bytes.data(), bytes.size());
-        int newTable = lua::gettop(L);
-
-        for (size_t i = 0; i < compressed_bytes.size(); i++) {
-            lua::pushinteger(L, compressed_bytes.data()[i]);
-            lua::rawseti(L, i + 1, newTable);
-        }
-        return 1;
     }
+    return 1;
 }
 
 static int l_gzip_decompress(lua::State* L) {
     std::vector<ubyte> bytes;
 
-    int result = read_bytes_from_table(L, -1, bytes);
+    read_bytes_from_table(L, 1, bytes);
+    auto decompressed_bytes = gzip::decompress(bytes.data(), bytes.size());
+    int newTable = lua::gettop(L);
 
-    if (result != 1) {
+    for (size_t i = 0; i < decompressed_bytes.size(); i++) {
-        return result;
+        lua::pushinteger(L, decompressed_bytes.data()[i]);
-    } else {
+        lua::rawseti(L, i + 1, newTable);
-        auto decompressed_bytes = gzip::decompress(bytes.data(), bytes.size());
-        int newTable = lua::gettop(L);
-
-        for (size_t i = 0; i < decompressed_bytes.size(); i++) {
-            lua::pushinteger(L, decompressed_bytes.data()[i]);
-            lua::rawseti(L, i + 1, newTable);
-        }
-        return 1;
     }
+    return 1;
 }
 
 static int l_read_combined_list(lua::State* L) {

src/logic/scripting/lua/libs/libnetwork.cpp

@@ -0,0 +1,185 @@
+#include "api_lua.hpp"
+
+#include "engine.hpp"
+#include "network/Network.hpp"
+
+using namespace scripting;
+
+static int l_get(lua::State* L) {
+    std::string url(lua::require_lstring(L, 1));
+
+    lua::pushvalue(L, 2);
+    auto onResponse = lua::create_lambda(L);
+
+    engine->getNetwork().get(url, [onResponse](std::vector<char> bytes) {
+        engine->postRunnable([=]() {
+            onResponse({std::string(bytes.data(), bytes.size())});
+        });
+    });
+    return 0;
+}
+
+static int l_get_binary(lua::State* L) {
+    std::string url(lua::require_lstring(L, 1));
+
+    lua::pushvalue(L, 2);
+    auto onResponse = lua::create_lambda(L);
+
+    engine->getNetwork().get(url, [onResponse](std::vector<char> bytes) {
+        auto buffer = std::make_shared<util::Buffer<ubyte>>(
+            reinterpret_cast<const ubyte*>(bytes.data()), bytes.size()
+        );
+        engine->postRunnable([=]() {
+            onResponse({buffer});
+        });
+    });
+    return 0;
+}
+
+static int l_connect(lua::State* L) {
+    std::string address = lua::require_string(L, 1);
+    int port = lua::tointeger(L, 2);
+    lua::pushvalue(L, 3);
+    auto callback = lua::create_lambda(L);
+    u64id_t id = engine->getNetwork().connect(address, port, [callback](u64id_t id) {
+        engine->postRunnable([=]() {
+            callback({id});
+        });
+    });
+    return lua::pushinteger(L, id);
+}
+
+
+static int l_close(lua::State* L) {
+    u64id_t id = lua::tointeger(L, 1);
+    if (auto connection = engine->getNetwork().getConnection(id)) {
+        connection->close();
+    }
+    return 0;
+}
+
+static int l_closeserver(lua::State* L) {
+    u64id_t id = lua::tointeger(L, 1);
+    if (auto server = engine->getNetwork().getServer(id)) {
+        server->close();
+    }
+    return 0;
+}
+
+static int l_send(lua::State* L) {
+    u64id_t id = lua::tointeger(L, 1);
+    auto connection = engine->getNetwork().getConnection(id);
+    if (connection == nullptr) {
+        return 0;
+    }
+    if (lua::istable(L, 2)) {
+        lua::pushvalue(L, 2);
+        size_t size = lua::objlen(L, 2);
+        util::Buffer<char> buffer(size);
+        for (size_t i = 0; i < size; i++) {
+            lua::rawgeti(L, i + 1);
+            buffer[i] = lua::tointeger(L, -1);
+            lua::pop(L);
+        }
+        lua::pop(L);
+        connection->send(buffer.data(), size);
+    } else if (auto bytes = lua::touserdata<lua::LuaBytearray>(L, 2)) {
+        connection->send(
+            reinterpret_cast<char*>(bytes->data().data()), bytes->data().size()
+        );
+    }
+    return 0;
+}
+
+static int l_recv(lua::State* L) {
+    u64id_t id = lua::tointeger(L, 1);
+    int length = lua::tointeger(L, 2);
+    auto connection = engine->getNetwork().getConnection(id);
+    if (connection == nullptr) {
+        return 0;
+    }
+    util::Buffer<char> buffer(glm::min(length, connection->available()));
+    
+    int size = connection->recv(buffer.data(), length);
+    if (size == -1) {
+        return 0;
+    }
+    if (lua::toboolean(L, 3)) {
+        lua::createtable(L, size, 0);
+        for (size_t i = 0; i < size; i++) {
+            lua::pushinteger(L, buffer[i] & 0xFF);
+            lua::rawseti(L, i+1);
+        }
+    } else {
+        lua::newuserdata<lua::LuaBytearray>(L, size);
+        auto bytearray = lua::touserdata<lua::LuaBytearray>(L, -1);   
+        bytearray->data().reserve(size);
+        std::memcpy(bytearray->data().data(), buffer.data(), size);
+    }
+    return 1;
+}
+
+static int l_open(lua::State* L) {
+    int port = lua::tointeger(L, 1);
+    lua::pushvalue(L, 2);
+    auto callback = lua::create_lambda(L);
+    u64id_t id = engine->getNetwork().openServer(port, [callback](u64id_t id) {
+        engine->postRunnable([=]() {
+            callback({id});
+        });
+    });
+    return lua::pushinteger(L, id);
+}
+
+static int l_is_alive(lua::State* L) {
+    u64id_t id = lua::tointeger(L, 1);
+    if (auto connection = engine->getNetwork().getConnection(id)) {
+        return lua::pushboolean(
+            L, connection->getState() != network::ConnectionState::CLOSED
+        );
+    }
+    return lua::pushboolean(L, false);
+}
+
+static int l_is_connected(lua::State* L) {
+    u64id_t id = lua::tointeger(L, 1);
+    if (auto connection = engine->getNetwork().getConnection(id)) {
+        return lua::pushboolean(
+            L, connection->getState() == network::ConnectionState::CONNECTED
+        );
+    }
+    return lua::pushboolean(L, false);
+}
+
+static int l_is_serveropen(lua::State* L) {
+    u64id_t id = lua::tointeger(L, 1);
+    if (auto server = engine->getNetwork().getServer(id)) {
+        return lua::pushboolean(L, server->isOpen());
+    }
+    return lua::pushboolean(L, false);
+}
+
+static int l_get_total_upload(lua::State* L) {
+    return lua::pushinteger(L, engine->getNetwork().getTotalUpload());
+}
+
+static int l_get_total_download(lua::State* L) {
+    return lua::pushinteger(L, engine->getNetwork().getTotalDownload());
+}
+
+const luaL_Reg networklib[] = {
+    {"get", lua::wrap<l_get>},
+    {"get_binary", lua::wrap<l_get_binary>},
+    {"get_total_upload", lua::wrap<l_get_total_upload>},
+    {"get_total_download", lua::wrap<l_get_total_download>},
+    {"__open", lua::wrap<l_open>},
+    {"__closeserver", lua::wrap<l_closeserver>},
+    {"__connect", lua::wrap<l_connect>},
+    {"__close", lua::wrap<l_close>},
+    {"__send", lua::wrap<l_send>},
+    {"__recv", lua::wrap<l_recv>},
+    {"__is_alive", lua::wrap<l_is_alive>},
+    {"__is_connected", lua::wrap<l_is_connected>},
+    {"__is_serveropen", lua::wrap<l_is_serveropen>},
+    {NULL, NULL}
+};

src/logic/scripting/lua/lua_engine.cpp

@@ -65,6 +65,7 @@ static void create_libs(State* L, StateType stateType) {
         openlib(L, "audio", audiolib);
         openlib(L, "console", consolelib);
         openlib(L, "player", playerlib);
+        openlib(L, "network", networklib);
 
         openlib(L, "entities", entitylib);
         openlib(L, "cameras", cameralib);

src/logic/scripting/scripting.cpp

@@ -208,23 +208,22 @@ void scripting::on_world_load(LevelController* controller) {
     if (lua::getglobal(L, "__vc_on_world_open")) {
         lua::call_nothrow(L, 0, 0);
     } 
-    load_script("world.lua", false);
     
-    for (auto& pack : scripting::engine->getContentPacks()) {
+    for (auto& pack : scripting::engine->getAllContentPacks()) {
         lua::emit_event(L, pack.id + ":.worldopen");
     }
 }
 
 void scripting::on_world_tick() {
     auto L = lua::get_main_state();
-    for (auto& pack : scripting::engine->getContentPacks()) {
+    for (auto& pack : scripting::engine->getAllContentPacks()) {
         lua::emit_event(L, pack.id + ":.worldtick");
     }
 }
 
 void scripting::on_world_save() {
     auto L = lua::get_main_state();
-    for (auto& pack : scripting::engine->getContentPacks()) {
+    for (auto& pack : scripting::engine->getAllContentPacks()) {
         lua::emit_event(L, pack.id + ":.worldsave");
     }
     if (lua::getglobal(L, "__vc_on_world_save")) {
@@ -234,7 +233,7 @@ void scripting::on_world_save() {
 
 void scripting::on_world_quit() {
     auto L = lua::get_main_state();
-    for (auto& pack : scripting::engine->getContentPacks()) {
+    for (auto& pack : scripting::engine->getAllContentPacks()) {
         lua::emit_event(L, pack.id + ":.worldquit");
     }
     if (lua::getglobal(L, "__vc_on_world_quit")) {

src/logic/scripting/scripting_hud.cpp

@@ -42,7 +42,7 @@ void scripting::on_frontend_init(Hud* hud, WorldRenderer* renderer) {
         lua::call_nothrow(L, 0, 0);
     }
 
-    for (auto& pack : engine->getContentPacks()) {
+    for (auto& pack : engine->getAllContentPacks()) {
         lua::emit_event(
             lua::get_main_state(),
             pack.id + ":.hudopen",
@@ -54,7 +54,7 @@ void scripting::on_frontend_init(Hud* hud, WorldRenderer* renderer) {
 }
 
 void scripting::on_frontend_render() {
-    for (auto& pack : engine->getContentPacks()) {
+    for (auto& pack : engine->getAllContentPacks()) {
         lua::emit_event(
             lua::get_main_state(),
             pack.id + ":.hudrender",
@@ -64,7 +64,7 @@ void scripting::on_frontend_render() {
 }
 
 void scripting::on_frontend_close() {
-    for (auto& pack : engine->getContentPacks()) {
+    for (auto& pack : engine->getAllContentPacks()) {
         lua::emit_event(
             lua::get_main_state(),
             pack.id + ":.hudclose",

src/network/Network.cpp

@@ -0,0 +1,622 @@
+#include "Network.hpp"
+
+#pragma comment(lib, "Ws2_32.lib")
+
+#define NOMINMAX
+#include <curl/curl.h>
+#include <stdexcept>
+#include <limits>
+#include <queue>
+#include <thread>
+
+#ifdef _WIN32
+/// included in curl.h
+#else
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+
+using SOCKET = int;
+#endif // _WIN32
+
+#include "debug/Logger.hpp"
+#include "util/stringutil.hpp"
+
+using namespace network;
+
+static debug::Logger logger("network");
+
+static size_t write_callback(
+    char* ptr, size_t size, size_t nmemb, void* userdata
+) {
+    auto& buffer = *reinterpret_cast<std::vector<char>*>(userdata);
+    size_t psize = buffer.size();
+    buffer.resize(psize + size * nmemb);
+    std::memcpy(buffer.data() + psize, ptr, size * nmemb);
+    return size * nmemb;
+}
+
+struct Request {
+    std::string url;
+    OnResponse onResponse;
+    OnReject onReject;
+    long maxSize;
+    bool followLocation = false;
+};
+
+class CurlRequests : public Requests {
+    CURLM* multiHandle;
+    CURL* curl;
+
+    size_t totalUpload = 0;
+    size_t totalDownload = 0;
+
+    OnResponse onResponse;
+    OnReject onReject;
+    std::vector<char> buffer;
+    std::string url;
+
+    std::queue<Request> requests;
+public:
+    CurlRequests(CURLM* multiHandle, CURL* curl)
+        : multiHandle(multiHandle), curl(curl) {
+    }
+
+    virtual ~CurlRequests() {
+        curl_multi_remove_handle(multiHandle, curl);
+        curl_easy_cleanup(curl);
+        curl_multi_cleanup(multiHandle);
+    }
+
+    void get(
+        const std::string& url,
+        OnResponse onResponse,
+        OnReject onReject,
+        long maxSize
+    ) override {
+        Request request {url, onResponse, onReject, maxSize};
+        if (url.empty()) {
+            processRequest(request);
+        } else {
+            requests.push(request);
+        }
+    }
+
+    void processRequest(const Request& request) {
+        onResponse = request.onResponse;
+        onReject = request.onReject;
+        url = request.url;
+
+        buffer.clear();
+
+        curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer);
+        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, request.followLocation);
+        if (request.maxSize == 0) {
+            curl_easy_setopt(
+                curl, CURLOPT_MAXFILESIZE, std::numeric_limits<long>::max()
+            );
+        } else {
+            curl_easy_setopt(curl, CURLOPT_MAXFILESIZE, request.maxSize);
+        }
+        curl_multi_add_handle(multiHandle, curl);
+        int running;
+        CURLMcode res = curl_multi_perform(multiHandle, &running);
+        if (res != CURLM_OK) {
+            auto message = curl_multi_strerror(res);
+            logger.error() << message << " (" << url << ")";
+            if (onReject) {
+                onReject(message);
+            }
+            url = "";
+        }
+    }
+
+    void update() override {
+        int messagesLeft;
+        int running;
+        CURLMsg* msg;
+        CURLMcode res = curl_multi_perform(multiHandle, &running);
+        if (res != CURLM_OK) {
+            auto message = curl_multi_strerror(res);
+            logger.error() << message << " (" << url << ")";
+            if (onReject) {
+                onReject(message);
+            }
+            curl_multi_remove_handle(multiHandle, curl);
+            url = "";
+            return;
+        }
+        if ((msg = curl_multi_info_read(multiHandle, &messagesLeft)) != NULL) {
+            if(msg->msg == CURLMSG_DONE) {
+                curl_multi_remove_handle(multiHandle, curl);
+            }
+            int response;
+            curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &response);
+            if (response == 200) {
+                long size;
+                if (!curl_easy_getinfo(curl, CURLINFO_REQUEST_SIZE, &size)) {
+                    totalUpload += size;
+                }
+                if (!curl_easy_getinfo(curl, CURLINFO_HEADER_SIZE, &size)) {
+                    totalDownload += size;
+                }
+                totalDownload += buffer.size();
+                if (onResponse) {
+                    onResponse(std::move(buffer));
+                }
+            } else {
+                logger.error() << "response code " << response << " (" << url << ")";
+                if (onReject) {
+                    onReject(std::to_string(response).c_str());
+                }
+            }
+            url = "";
+        }
+        if (url.empty() && !requests.empty()) {
+            auto request = std::move(requests.front());
+            requests.pop();
+            processRequest(request);
+        }
+    }
+
+    size_t getTotalUpload() const override {
+        return totalUpload;
+    }
+
+    size_t getTotalDownload() const override {
+        return totalDownload;
+    }
+
+    static std::unique_ptr<CurlRequests> create() {
+        auto curl = curl_easy_init();
+        if (curl == nullptr) {
+            throw std::runtime_error("could not initialzie cURL");
+        }
+        auto multiHandle = curl_multi_init();
+        if (multiHandle == nullptr) {
+            curl_easy_cleanup(curl);
+            throw std::runtime_error("could not initialzie cURL-multi");
+        }
+        return std::make_unique<CurlRequests>(multiHandle, curl);
+    }
+};
+
+#ifndef _WIN32
+static inline int closesocket(int descriptor) noexcept {
+    return close(descriptor);
+}
+static inline std::runtime_error handle_socket_error(const std::string& message) {
+    int err = errno;
+    return std::runtime_error(
+        message+" [errno=" + std::to_string(err) + "]: " + 
+        std::string(strerror(err))
+    );
+}
+#else
+static inline std::runtime_error handle_socket_error(const std::string& message) {
+    int errorCode = WSAGetLastError();
+    wchar_t* s = nullptr;
+    size_t size = FormatMessageW(
+        FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
+            FORMAT_MESSAGE_IGNORE_INSERTS,
+        nullptr,
+        errorCode,
+        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+        (LPWSTR)&s,
+        0,
+        nullptr
+    );
+    assert(s != nullptr);
+    while (size && isspace(s[size-1])) {
+        s[--size] = 0;
+    }
+    auto errorString = util::wstr2str_utf8(std::wstring(s));
+    LocalFree(s);
+    return std::runtime_error(message+" [WSA error=" + 
+           std::to_string(errorCode) + "]: "+errorString);
+}
+#endif
+
+static inline int connectsocket(
+    int descriptor, const sockaddr* addr, socklen_t len
+) noexcept {
+    return connect(descriptor, addr, len);
+}
+
+static inline int recvsocket(
+    int descriptor, char* buf, size_t len
+) noexcept {
+    return recv(descriptor, buf, len, 0);
+}
+
+static inline int sendsocket(
+    int descriptor, const char* buf, size_t len, int flags
+) noexcept {
+    return send(descriptor, buf, len, flags);
+}
+
+static std::string to_string(const sockaddr_in* addr) {
+    char ip[INET_ADDRSTRLEN];
+    if (inet_ntop(AF_INET, &(addr->sin_addr), ip, INET_ADDRSTRLEN)) {
+        return std::string(ip)+":"+std::to_string(htons(addr->sin_port));
+    }
+    return "";
+}
+
+class SocketConnection : public Connection {
+    SOCKET descriptor;
+    bool open = true;
+    sockaddr_in addr;
+    size_t totalUpload = 0;
+    size_t totalDownload = 0;
+    ConnectionState state = ConnectionState::INITIAL;
+    std::unique_ptr<std::thread> thread = nullptr;
+    std::vector<char> readBatch;
+    util::Buffer<char> buffer;
+    std::mutex mutex;
+
+    void connectSocket() {
+        state = ConnectionState::CONNECTING;
+        logger.info() << "connecting to " << to_string(&addr);
+        int res = connectsocket(descriptor, (const sockaddr*)&addr, sizeof(sockaddr_in));
+        if (res < 0) {
+            auto error = handle_socket_error("Connect failed");
+            closesocket(descriptor);
+            state = ConnectionState::CLOSED;
+            logger.error() << error.what();
+            return;
+        }
+        logger.info() << "connected to " << to_string(&addr);
+        state = ConnectionState::CONNECTED;
+    }
+public:
+    SocketConnection(SOCKET descriptor, sockaddr_in addr)
+        : descriptor(descriptor), addr(std::move(addr)), buffer(16'384) {}
+
+    ~SocketConnection() {
+        if (state != ConnectionState::CLOSED) {
+            shutdown(descriptor, 2);
+            closesocket(descriptor);
+        }
+        if (thread) {
+            thread->join();
+        }
+    }
+
+    void connect(runnable callback) override {
+        thread = std::make_unique<std::thread>([this, callback]() {
+            connectSocket();
+            if (state == ConnectionState::CONNECTED) {
+                callback();
+            }
+            while (state == ConnectionState::CONNECTED) {
+                int size = recvsocket(descriptor, buffer.data(), buffer.size());
+                if (size == 0) {
+                    logger.info() << "closed connection with " << to_string(&addr);
+                    closesocket(descriptor);
+                    state = ConnectionState::CLOSED;
+                    break;
+                } else if (size < 0) {
+                    logger.info() << "an error ocurred while receiving from "
+                                  << to_string(&addr);
+                    auto error = handle_socket_error("recv(...) error");
+                    closesocket(descriptor);
+                    state = ConnectionState::CLOSED;
+                    logger.error() << error.what();
+                    break;
+                }
+                {
+                    std::lock_guard lock(mutex);
+                    for (size_t i = 0; i < size; i++) {
+                        readBatch.emplace_back(buffer[i]);
+                    }
+                    totalDownload += size;
+                }
+                logger.info() << "read " << size << " bytes from " << to_string(&addr);
+            }
+        });
+    }
+
+    int recv(char* buffer, size_t length) override {
+        std::lock_guard lock(mutex);
+
+        if (state != ConnectionState::CONNECTED && readBatch.empty()) {
+            return -1;
+        }
+        int size = std::min(readBatch.size(), length);
+        std::memcpy(buffer, readBatch.data(), size);
+        readBatch.erase(readBatch.begin(), readBatch.begin() + size);
+        return size;
+    }
+
+    int send(const char* buffer, size_t length) override {
+        int len = sendsocket(descriptor, buffer, length, 0);
+        if (len == -1) {
+            int err = errno;
+            close();
+            throw std::runtime_error(
+                "Send failed [errno=" + std::to_string(err) + "]: "
+                 + std::string(strerror(err))
+            );
+        }
+        totalUpload += len;
+        return len;
+    }
+
+    int available() override {
+        std::lock_guard lock(mutex);
+        return readBatch.size();
+    }
+
+    void close() override {
+        if (state != ConnectionState::CLOSED) {
+            shutdown(descriptor, 2);
+            closesocket(descriptor);
+        }
+        if (thread) {
+            thread->join();
+            thread = nullptr;
+        }
+    }
+
+    size_t pullUpload() override {
+        size_t size = totalUpload;
+        totalUpload = 0;
+        return size;
+    }
+
+    size_t pullDownload() override {
+        size_t size = totalDownload;
+        totalDownload = 0;
+        return size;
+    }
+
+    static std::shared_ptr<SocketConnection> connect(
+        const std::string& address, int port, runnable callback
+    ) {
+        addrinfo hints {};
+
+        hints.ai_family = AF_INET;
+        hints.ai_socktype = SOCK_STREAM;
+
+        addrinfo* addrinfo;
+        if (int res = getaddrinfo(
+            address.c_str(), nullptr, &hints, &addrinfo
+        )) {
+            throw std::runtime_error(gai_strerror(res));
+        }
+
+        sockaddr_in serverAddress = *(sockaddr_in*)addrinfo->ai_addr;
+        freeaddrinfo(addrinfo);
+        serverAddress.sin_port = htons(port);
+
+        SOCKET descriptor = socket(AF_INET, SOCK_STREAM, 0);
+        if (descriptor == -1) {
+            freeaddrinfo(addrinfo);
+            throw std::runtime_error("Could not create socket");
+        }
+        auto socket = std::make_shared<SocketConnection>(descriptor, std::move(serverAddress));
+        socket->connect(std::move(callback));
+        return socket;
+    }
+
+    ConnectionState getState() const override {
+        return state;
+    }
+};
+
+class SocketTcpSServer : public TcpServer {
+    Network* network;
+    SOCKET descriptor;
+    std::vector<u64id_t> clients;
+    std::mutex clientsMutex;
+    bool open = true;
+    std::unique_ptr<std::thread> thread = nullptr;
+public:
+    SocketTcpSServer(Network* network, SOCKET descriptor)
+    : network(network), descriptor(descriptor) {}
+
+    ~SocketTcpSServer() {
+        closeSocket();
+    }
+
+    void startListen(consumer<u64id_t> handler) override {
+        thread = std::make_unique<std::thread>([this, handler]() {
+            while (open) {
+                logger.info() << "listening for connections";
+                if (listen(descriptor, 2) < 0) {
+                    close();
+                    break;
+                }
+                socklen_t addrlen = sizeof(sockaddr_in);
+                SOCKET clientDescriptor;
+                sockaddr_in address;
+                logger.info() << "accepting clients";
+                if ((clientDescriptor = accept(descriptor, (sockaddr*)&address, &addrlen)) == -1) {
+                    close();
+                    break;
+                }
+                logger.info() << "client connected: " << to_string(&address);
+                auto socket = std::make_shared<SocketConnection>(
+                    clientDescriptor, address
+                );
+                u64id_t id = network->addConnection(socket);
+                {
+                    std::lock_guard lock(clientsMutex);
+                    clients.push_back(id);
+                }
+                handler(id);
+            }
+        });
+    }
+    
+    void closeSocket() {
+        if (!open) {
+            return;
+        }
+        logger.info() << "closing server";
+        open = false;
+
+        {
+            std::lock_guard lock(clientsMutex);
+            for (u64id_t clientid : clients) {
+                if (auto client = network->getConnection(clientid)) {
+                    client->close();
+                }
+            }
+        }
+        clients.clear();
+
+        shutdown(descriptor, 2);
+        closesocket(descriptor);
+        thread->join();
+    }
+
+    void close() override {
+        closeSocket();
+    }
+    
+    bool isOpen() override {
+        return open;
+    }
+    static std::shared_ptr<SocketTcpSServer> openServer(
+        Network* network, int port, consumer<u64id_t> handler
+    ) {
+        SOCKET descriptor = socket(
+            AF_INET, SOCK_STREAM, 0
+        );
+        if (descriptor == -1) {
+            throw std::runtime_error("Could not create server socket");
+        }
+        int opt = 1;
+        int flags = SO_REUSEADDR;
+#       ifndef _WIN32
+            flags |= SO_REUSEPORT;
+#       endif
+        if (setsockopt(descriptor, SOL_SOCKET, flags, (const char*)&opt, sizeof(opt))) {
+            closesocket(descriptor);
+            throw std::runtime_error("setsockopt");
+        }
+        sockaddr_in address;
+        address.sin_family = AF_INET;
+        address.sin_addr.s_addr = INADDR_ANY;
+        address.sin_port = htons(port);
+        if (bind(descriptor, (sockaddr*)&address, sizeof(address)) < 0) {
+            closesocket(descriptor);
+            throw std::runtime_error("could not bind port "+std::to_string(port));
+        }
+        logger.info() << "opened server at port " << port;
+        auto server = std::make_shared<SocketTcpSServer>(network, descriptor);
+        server->startListen(std::move(handler));
+        return server;
+    }
+};
+
+Network::Network(std::unique_ptr<Requests> requests)
+: requests(std::move(requests)) {
+}
+
+Network::~Network() = default;
+
+void Network::get(
+    const std::string& url,
+    OnResponse onResponse,
+    OnReject onReject,
+    long maxSize
+) {
+    requests->get(url, onResponse, onReject, maxSize);
+}
+
+Connection* Network::getConnection(u64id_t id) {
+    std::lock_guard lock(connectionsMutex);
+
+    const auto& found = connections.find(id);
+    if (found == connections.end()) {
+        return nullptr;
+    }
+    return found->second.get();
+}
+
+TcpServer* Network::getServer(u64id_t id) const {
+    const auto& found = servers.find(id);
+    if (found == servers.end()) {
+        return nullptr;
+    }
+    return found->second.get();
+}
+
+u64id_t Network::connect(const std::string& address, int port, consumer<u64id_t> callback) {
+    std::lock_guard lock(connectionsMutex);
+    
+    u64id_t id = nextConnection++;
+    auto socket = SocketConnection::connect(address, port, [id, callback]() {
+        callback(id);
+    });
+    connections[id] = std::move(socket);
+    return id;
+}
+
+u64id_t Network::openServer(int port, consumer<u64id_t> handler) {
+    u64id_t id = nextServer++;
+    auto server = SocketTcpSServer::openServer(this, port, handler);
+    servers[id] = std::move(server);
+    return id;
+}
+
+u64id_t Network::addConnection(const std::shared_ptr<Connection>& socket) {
+    std::lock_guard lock(connectionsMutex);
+
+    u64id_t id = nextConnection++;
+    connections[id] = std::move(socket);
+    return id;
+}
+
+size_t Network::getTotalUpload() const {
+    return requests->getTotalUpload() + totalUpload;
+}
+
+size_t Network::getTotalDownload() const {
+    return requests->getTotalDownload() + totalDownload;
+}
+
+void Network::update() {
+    requests->update();
+
+    {
+        std::lock_guard lock(connectionsMutex);
+        auto socketiter = connections.begin();
+        while (socketiter != connections.end()) {
+            auto socket = socketiter->second.get();
+            totalDownload += socket->pullDownload();
+            totalUpload += socket->pullUpload();
+            if (socket->available() == 0 && 
+                socket->getState() == ConnectionState::CLOSED) {
+                socketiter = connections.erase(socketiter);
+                continue;
+            }
+            ++socketiter;
+        }
+        auto serveriter = servers.begin();
+        while (serveriter != servers.end()) {
+            auto server = serveriter->second.get();
+            if (!server->isOpen()) {
+                serveriter = servers.erase(serveriter);
+                continue;
+            }
+            ++serveriter;
+        }
+    }
+}
+
+std::unique_ptr<Network> Network::create(const NetworkSettings& settings) {
+    auto requests = CurlRequests::create();
+    return std::make_unique<Network>(std::move(requests));
+}

src/network/Network.hpp

@@ -0,0 +1,99 @@
+#pragma once
+
+#include <memory>
+#include <vector>
+#include <mutex>
+
+#include "typedefs.hpp"
+#include "settings.hpp"
+#include "util/Buffer.hpp"
+#include "delegates.hpp"
+
+namespace network {
+    using OnResponse = std::function<void(std::vector<char>)>;
+    using OnReject = std::function<void(const char*)>;
+
+    class Requests {
+    public:
+        virtual ~Requests() {}
+
+        virtual void get(
+            const std::string& url,
+            OnResponse onResponse,
+            OnReject onReject=nullptr,
+            long maxSize=0
+        ) = 0;
+        virtual size_t getTotalUpload() const = 0;
+        virtual size_t getTotalDownload() const = 0;
+
+        virtual void update() = 0;
+    };
+
+    enum class ConnectionState {
+        INITIAL, CONNECTING, CONNECTED, CLOSED
+    };
+
+    class Connection {
+    public:
+        virtual ~Connection() {}
+
+        virtual void connect(runnable callback) = 0;
+        virtual int recv(char* buffer, size_t length) = 0;
+        virtual int send(const char* buffer, size_t length) = 0;
+        virtual void close() = 0;
+        virtual int available() = 0;
+
+        virtual size_t pullUpload() = 0;
+        virtual size_t pullDownload() = 0;
+
+        virtual ConnectionState getState() const = 0;
+    };
+
+    class TcpServer {
+    public:
+        virtual ~TcpServer() {}
+        virtual void startListen(consumer<u64id_t> handler) = 0;
+        virtual void close() = 0;
+        virtual bool isOpen() = 0;
+    };
+
+    class Network {
+        std::unique_ptr<Requests> requests;
+
+        std::unordered_map<u64id_t, std::shared_ptr<Connection>> connections;
+        std::mutex connectionsMutex {};
+        u64id_t nextConnection = 1;
+
+        std::unordered_map<u64id_t, std::shared_ptr<TcpServer>> servers;
+        u64id_t nextServer = 1;
+
+        size_t totalDownload = 0;
+        size_t totalUpload = 0;
+    public:
+        Network(std::unique_ptr<Requests> requests);
+        ~Network();
+
+        void get(
+            const std::string& url,
+            OnResponse onResponse,
+            OnReject onReject = nullptr,
+            long maxSize=0
+        );
+
+        [[nodiscard]] Connection* getConnection(u64id_t id);
+        [[nodiscard]] TcpServer* getServer(u64id_t id) const;
+
+        u64id_t connect(const std::string& address, int port, consumer<u64id_t> callback);
+
+        u64id_t openServer(int port, consumer<u64id_t> handler);
+
+        u64id_t addConnection(const std::shared_ptr<Connection>& connection);
+
+        size_t getTotalUpload() const;
+        size_t getTotalDownload() const;
+
+        void update();
+
+        static std::unique_ptr<Network> create(const NetworkSettings& settings);
+    };
+}

src/settings.hpp

@@ -81,6 +81,9 @@ struct UiSettings {
     IntegerSetting worldPreviewSize {64, 1, 512};
 };
 
+struct NetworkSettings {
+};
+
 struct EngineSettings {
     AudioSettings audio;
     DisplaySettings display;
@@ -89,4 +92,5 @@ struct EngineSettings {
     GraphicsSettings graphics;
     DebugSettings debug;
     UiSettings ui;
+    NetworkSettings network;
 };

src/util/stringutil.cpp

@@ -7,7 +7,7 @@
 #include <sstream>
 #include <stdexcept>
 
-std::string util::escape(std::string_view s) {
+std::string util::escape(std::string_view s, bool escapeUnicode) {
     std::stringstream ss;
     ss << '"';
     size_t pos = 0;
@@ -39,8 +39,12 @@ std::string util::escape(std::string_view s) {
                 if (c & 0x80) {
                     uint cpsize;
                     int codepoint = decode_utf8(cpsize, s.data() + pos);
+                    if (escapeUnicode) {
+                        ss << "\\u" << std::hex << codepoint;
+                    } else {
+                        ss << std::string(s.data() + pos, cpsize);
+                    }
                     pos += cpsize-1;
-                    ss << "\\u" << std::hex << codepoint;
                     break;
                 }
                 if (c < ' ') {
@@ -57,7 +61,7 @@ std::string util::escape(std::string_view s) {
 }
 
 std::string util::quote(const std::string& s) {
-    return escape(s);
+    return escape(s, false);
 }
 
 std::wstring util::lfill(std::wstring s, uint length, wchar_t c) {

src/util/stringutil.hpp

@@ -8,7 +8,7 @@
 
 namespace util {
     /// @brief Function used for string serialization in text formats
-    std::string escape(std::string_view s);
+    std::string escape(std::string_view s, bool escapeUnicode=true);
 
     /// @brief Function used for error messages
     std::string quote(const std::string& s);

test/network/curltest.cpp

@@ -0,0 +1,23 @@
+#include <gtest/gtest.h>
+
+#include "network/Network.hpp"
+#include "coders/json.hpp"
+
+TEST(curltest, curltest) {
+    NetworkSettings settings {};
+    auto network = network::Network::create(settings);
+    network->get(
+        "https://raw.githubusercontent.com/MihailRis/VoxelEngine-Cpp/refs/"
+        "heads/curl/res/content/base/blocks/lamp.json",
+        [](std::vector<char> data) {
+            if (data.empty()) {
+                return;
+            }
+            auto view = std::string_view(data.data(), data.size());
+            auto value = json::parse(view);
+            std::cout << value << std::endl;
+        }, [](auto){}
+    );
+    std::cout << "upload: " << network->getTotalUpload() << " B" << std::endl;
+    std::cout << "download: " << network->getTotalDownload() << " B" << std::endl;
+}

vcpkg.json

@@ -12,6 +12,7 @@
       "luajit",
       "libvorbis",
       "entt",
-      "gtest"
+      "gtest",
+      "curl"
     ]
   }