PulseStudio/VoxelEngine
PulseStudio/VoxelEngine
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

regions related code moved from WorldFiles to WorldRegions

Browse Source
This commit is contained in:
MihailRis 2024-04-15 03:45:55 +03:00
parent d2a2f0e8ae
commit 3ebd8f3062
8 changed files with 641 additions and 576 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/files/WorldConverter.cpp
View File

@ -24,7 +24,7 @@ WorldConverter::WorldConverter(
lut(lut),
content(content)
{
fs::path regionsFolder = wfile->getRegionsFolder(REGION_LAYER_VOXELS);
fs::path regionsFolder = wfile->getRegions().getRegionsFolder(REGION_LAYER_VOXELS);
if (!fs::is_directory(regionsFolder)) {
logger.error() << "nothing to convert";
return;
@ -46,7 +46,7 @@ void WorldConverter::convertRegion(fs::path file) {
return;
}
logger.info() << "converting region " << name;
wfile->processRegionVoxels(x, z, [=](ubyte* data) {
wfile->getRegions().processRegionVoxels(x, z, [=](ubyte* data) {
if (lut) {
Chunk::convert(data, lut.get());
}

446
src/files/WorldFiles.cpp
View File

@ -2,7 +2,6 @@
#include "../coders/byte_utils.h"
#include "../coders/json.h"
#include "../coders/rle.h"
#include "../constants.h"
#include "../content/Content.h"
#include "../core_defs.h"
@ -28,86 +27,20 @@
#include <sstream>
#include <cstring>
#define REGION_FORMAT_MAGIC ".VOXREG"
#define WORLD_FORMAT_MAGIC ".VOXWLD"
const size_t BUFFER_SIZE_UNKNOWN = -1;
regfile::regfile(fs::path filename) : file(filename) {
if (file.length() < REGION_HEADER_SIZE)
throw std::runtime_error("incomplete region file header");
char header[REGION_HEADER_SIZE];
file.read(header, REGION_HEADER_SIZE);
// avoid of use strcmp_s
if (std::string(header, strlen(REGION_FORMAT_MAGIC)) != REGION_FORMAT_MAGIC) {
throw std::runtime_error("invalid region file magic number");
}
version = header[8];
if (uint(version) > REGION_FORMAT_VERSION) {
throw illegal_region_format(
"region format "+std::to_string(version)+" is not supported"
);
}
WorldFiles::WorldFiles(fs::path directory) : directory(directory), regions(directory) {
}
WorldRegion::WorldRegion() {
chunksData = new ubyte*[REGION_CHUNKS_COUNT]{};
sizes = new uint32_t[REGION_CHUNKS_COUNT]{};
}
WorldRegion::~WorldRegion() {
for (uint i = 0; i < REGION_CHUNKS_COUNT; i++) {
delete[] chunksData[i];
}
delete[] sizes;
delete[] chunksData;
}
void WorldRegion::setUnsaved(bool unsaved) {
this->unsaved = unsaved;
}
bool WorldRegion::isUnsaved() const {
return unsaved;
}
ubyte** WorldRegion::getChunks() const {
return chunksData;
}
uint32_t* WorldRegion::getSizes() const {
return sizes;
}
void WorldRegion::put(uint x, uint z, ubyte* data, uint32_t size) {
size_t chunk_index = z * REGION_SIZE + x;
delete[] chunksData[chunk_index];
chunksData[chunk_index] = data;
sizes[chunk_index] = size;
}
ubyte* WorldRegion::getChunkData(uint x, uint z) {
return chunksData[z * REGION_SIZE + x];
}
uint WorldRegion::getChunkDataSize(uint x, uint z) {
return sizes[z * REGION_SIZE + x];
}
WorldFiles::WorldFiles(fs::path directory) : directory(directory) {
for (uint i = 0; i < sizeof(layers)/sizeof(RegionsLayer); i++) {
layers[i].layer = i;
}
layers[REGION_LAYER_VOXELS].folder = directory/fs::path("regions");
layers[REGION_LAYER_LIGHTS].folder = directory/fs::path("lights");
layers[REGION_LAYER_INVENTORIES].folder = directory/fs::path("inventories");
}
WorldFiles::WorldFiles(fs::path directory, const DebugSettings& settings)
WorldFiles::WorldFiles(fs::path directory, const DebugSettings& settings)
: WorldFiles(directory)
{
generatorTestMode = settings.generatorTestMode;
doWriteLights = settings.doWriteLights;
regions.generatorTestMode = generatorTestMode;
regions.doWriteLights = doWriteLights;
}
WorldFiles::~WorldFiles() {
@ -118,115 +51,6 @@ void WorldFiles::createDirectories() {
fs::create_directories(directory / fs::path("content"));
}
WorldRegion* WorldFiles::getRegion(int x, int z, int layer) {
RegionsLayer& regions = layers[layer];
std::lock_guard lock(regions.mutex);
auto found = regions.regions.find(glm::ivec2(x, z));
if (found == regions.regions.end())
return nullptr;
return found->second.get();
}
WorldRegion* WorldFiles::getOrCreateRegion(int x, int z, int layer) {
RegionsLayer& regions = layers[layer];
WorldRegion* region = getRegion(x, z, layer);
if (region == nullptr) {
std::lock_guard lock(regions.mutex);
region = new WorldRegion();
regions.regions[glm::ivec2(x, z)].reset(region);
}
return region;
}
std::unique_ptr<ubyte[]> WorldFiles::compress(const ubyte* src, size_t srclen, size_t& len) {
auto buffer = bufferPool.get();
ubyte* bytes = buffer.get();
len = extrle::encode(src, srclen, bytes);
auto data = std::make_unique<ubyte[]>(len);
for (size_t i = 0; i < len; i++) {
data[i] = bytes[i];
}
return data;
}
std::unique_ptr<ubyte[]> WorldFiles::decompress(const ubyte* src, size_t srclen, size_t dstlen) {
auto decompressed = std::make_unique<ubyte[]>(dstlen);
extrle::decode(src, srclen, decompressed.get());
return decompressed;
}
inline void calc_reg_coords(
int x, int z, int& regionX, int& regionZ, int& localX, int& localZ
) {
regionX = floordiv(x, REGION_SIZE);
regionZ = floordiv(z, REGION_SIZE);
localX = x - (regionX * REGION_SIZE);
localZ = z - (regionZ * REGION_SIZE);
}
void WorldFiles::put(int x, int z, int layer, std::unique_ptr<ubyte[]> data, size_t size, bool rle) {
if (rle) {
size_t compressedSize;
auto compressed = compress(data.get(), size, compressedSize);
put(x, z, layer, std::move(compressed), compressedSize, false);
return;
}
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer);
region->setUnsaved(true);
region->put(localX, localZ, data.release(), size);
}
std::unique_ptr<ubyte[]> write_inventories(Chunk* chunk, uint& datasize) {
auto& inventories = chunk->inventories;
ByteBuilder builder;
builder.putInt32(inventories.size());
for (auto& entry : inventories) {
builder.putInt32(entry.first);
auto map = entry.second->serialize();
auto bytes = json::to_binary(map.get(), true);
builder.putInt32(bytes.size());
builder.put(bytes.data(), bytes.size());
}
auto datavec = builder.data();
datasize = builder.size();
auto data = std::make_unique<ubyte[]>(datasize);
for (uint i = 0; i < datasize; i++) {
data[i] = datavec[i];
}
return data;
}
/// @brief Store chunk (voxels and lights) in region (existing or new)
void WorldFiles::put(Chunk* chunk){
assert(chunk != nullptr);
int regionX, regionZ, localX, localZ;
calc_reg_coords(chunk->x, chunk->z, regionX, regionZ, localX, localZ);
put(chunk->x, chunk->z, REGION_LAYER_VOXELS,
chunk->encode(), CHUNK_DATA_LEN, true);
// Writing lights cache
if (doWriteLights && chunk->isLighted()) {
put(chunk->x, chunk->z, REGION_LAYER_LIGHTS,
chunk->lightmap.encode(), LIGHTMAP_DATA_LEN, true);
}
// Writing block inventories
if (!chunk->inventories.empty()){
uint datasize;
put(chunk->x, chunk->z, REGION_LAYER_INVENTORIES,
write_inventories(chunk, datasize), datasize, false);
}
}
fs::path WorldFiles::getRegionFilename(int x, int z) const {
return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin");
}
/// @brief Extract X and Z from 'X_Z.bin' region file name.
/// @param name source region file name
/// @param x parsed X destination
@ -263,237 +87,7 @@ fs::path WorldFiles::getPacksFile() const {
return directory/fs::path("packs.list");
}
std::unique_ptr<ubyte[]> WorldFiles::getChunk(int x, int z){
uint32_t size;
auto* data = getData(x, z, REGION_LAYER_VOXELS, size);
if (data == nullptr) {
return nullptr;
}
return decompress(data, size, CHUNK_DATA_LEN);
}
/// @brief Get cached lights for chunk at x,z
/// @return lights data or nullptr
std::unique_ptr<light_t[]> WorldFiles::getLights(int x, int z) {
uint32_t size;
auto* bytes = getData(x, z, REGION_LAYER_LIGHTS, size);
if (bytes == nullptr)
return nullptr;
auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN);
return Lightmap::decode(data.get());
}
chunk_inventories_map WorldFiles::fetchInventories(int x, int z) {
chunk_inventories_map inventories;
uint32_t bytesSize;
const ubyte* data = getData(x, z, REGION_LAYER_INVENTORIES, bytesSize);
if (data == nullptr)
return inventories;
ByteReader reader(data, bytesSize);
int count = reader.getInt32();
for (int i = 0; i < count; i++) {
uint index = reader.getInt32();
uint size = reader.getInt32();
auto map = json::from_binary(reader.pointer(), size);
reader.skip(size);
auto inv = std::make_shared<Inventory>(0, 0);
inv->deserialize(map.get());
inventories[index] = inv;
}
return inventories;
}
ubyte* WorldFiles::getData(
int x, int z, int layer,
uint32_t& size
) {
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer);
ubyte* data = region->getChunkData(localX, localZ);
if (data == nullptr) {
auto regfile = getRegFile(glm::ivec3(regionX, regionZ, layer));
if (regfile != nullptr) {
data = readChunkData(x, z, size, regfile.get()).release();
}
if (data != nullptr) {
region->put(localX, localZ, data, size);
}
}
if (data != nullptr) {
size = region->getChunkDataSize(localX, localZ);
return data;
}
return nullptr;
}
std::shared_ptr<regfile> WorldFiles::useRegFile(glm::ivec3 coord) {
return std::shared_ptr<regfile>(openRegFiles[coord].get(), [this](regfile* ptr) {
ptr->inUse = false;
regFilesCv.notify_one();
});
}
void WorldFiles::closeRegFile(glm::ivec3 coord) {
openRegFiles.erase(coord);
regFilesCv.notify_one();
}
// Marks regfile as used and unmarks when shared_ptr dies
std::shared_ptr<regfile> WorldFiles::getRegFile(glm::ivec3 coord) {
{
std::lock_guard lock(regFilesMutex);
const auto found = openRegFiles.find(coord);
if (found != openRegFiles.end()) {
if (found->second->inUse) {
throw std::runtime_error("regfile is currently in use");
}
found->second->inUse = true;
return useRegFile(found->first);
}
}
return createRegFile(coord);
}
std::shared_ptr<regfile> WorldFiles::createRegFile(glm::ivec3 coord) {
fs::path file = layers[coord[2]].folder/getRegionFilename(coord[0], coord[1]);
if (!fs::exists(file)) {
return nullptr;
}
if (openRegFiles.size() == MAX_OPEN_REGION_FILES) {
std::unique_lock lock(regFilesMutex);
while (true) {
bool closed = false;
// FIXME: bad choosing algorithm
for (auto& entry : openRegFiles) {
if (!entry.second->inUse) {
closeRegFile(entry.first);
closed = true;
break;
}
}
if (closed) {
break;
}
// notified when any regfile gets out of use or closed
regFilesCv.wait(lock);
}
openRegFiles[coord] = std::make_unique<regfile>(file);
return useRegFile(coord);
} else {
std::lock_guard lock(regFilesMutex);
openRegFiles[coord] = std::make_unique<regfile>(file);
return useRegFile(coord);
}
}
std::unique_ptr<ubyte[]> WorldFiles::readChunkData(
int x,
int z,
uint32_t& length,
regfile* rfile
){
if (generatorTestMode)
return nullptr;
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
int chunkIndex = localZ * REGION_SIZE + localX;
files::rafile& file = rfile->file;
size_t file_size = file.length();
size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4;
uint32_t offset;
file.seekg(table_offset + chunkIndex * 4);
file.read((char*)(&offset), 4);
offset = dataio::read_int32_big((const ubyte*)(&offset), 0);
if (offset == 0){
return nullptr;
}
file.seekg(offset);
file.read((char*)(&offset), 4);
length = dataio::read_int32_big((const ubyte*)(&offset), 0);
auto data = std::make_unique<ubyte[]>(length);
file.read((char*)data.get(), length);
return data;
}
/// @brief Read missing chunks data (null pointers) from region file
void WorldFiles::fetchChunks(WorldRegion* region, int x, int z, regfile* file) {
ubyte** chunks = region->getChunks();
uint32_t* sizes = region->getSizes();
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE;
int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE;
if (chunks[i] == nullptr) {
chunks[i] = readChunkData(chunk_x, chunk_z, sizes[i], file).release();
}
}
}
void WorldFiles::writeRegion(int x, int z, int layer, WorldRegion* entry){
fs::path filename = layers[layer].folder/getRegionFilename(x, z);
glm::ivec3 regcoord(x, z, layer);
if (auto regfile = getRegFile(regcoord)) {
fetchChunks(entry, x, z, regfile.get());
std::lock_guard lock(regFilesMutex);
closeRegFile(regcoord);
}
char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC;
header[8] = REGION_FORMAT_VERSION;
header[9] = 0; // flags
std::ofstream file(filename, std::ios::out | std::ios::binary);
file.write(header, REGION_HEADER_SIZE);
size_t offset = REGION_HEADER_SIZE;
char intbuf[4]{};
uint offsets[REGION_CHUNKS_COUNT]{};
ubyte** region = entry->getChunks();
uint32_t* sizes = entry->getSizes();
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
ubyte* chunk = region[i];
if (chunk == nullptr){
offsets[i] = 0;
} else {
offsets[i] = offset;
size_t compressedSize = sizes[i];
dataio::write_int32_big(compressedSize, (ubyte*)intbuf, 0);
offset += 4 + compressedSize;
file.write(intbuf, 4);
file.write((const char*)chunk, compressedSize);
}
}
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
dataio::write_int32_big(offsets[i], (ubyte*)intbuf, 0);
file.write(intbuf, 4);
}
}
void WorldFiles::writeRegions(int layer) {
for (auto& it : layers[layer].regions){
WorldRegion* region = it.second.get();
if (region->getChunks() == nullptr || !region->isUnsaved())
continue;
glm::ivec2 key = it.first;
writeRegion(key[0], key[1], layer, region);
}
}
void WorldFiles::write(const World* world, const Content* content) {
for (auto& layer : layers) {
fs::create_directories(layer.folder);
}
if (world) {
writeWorldInfo(world);
if (!fs::exists(getPacksFile())) {
@ -505,9 +99,7 @@ void WorldFiles::write(const World* world, const Content* content) {
}
writeIndices(content->getIndices());
for (auto& layer : layers) {
writeRegions(layer.layer);
}
regions.write();
}
void WorldFiles::writePacks(const std::vector<ContentPack>& packs) {
@ -585,34 +177,6 @@ void WorldFiles::removeIndices(const std::vector<std::string>& packs) {
files::write_json(getIndicesFile(), root.get());
}
void WorldFiles::processRegionVoxels(int x, int z, regionproc func) {
if (getRegion(x, z, REGION_LAYER_VOXELS)) {
throw std::runtime_error("not implemented for in-memory regions");
}
auto regfile = getRegFile(glm::ivec3(x, z, REGION_LAYER_VOXELS));
if (regfile == nullptr) {
throw std::runtime_error("could not open region file");
}
for (uint cz = 0; cz < REGION_SIZE; cz++) {
for (uint cx = 0; cx < REGION_SIZE; cx++) {
int gx = cx + x * REGION_SIZE;
int gz = cz + z * REGION_SIZE;
uint32_t length;
auto data = readChunkData(gx, gz, length, regfile.get());
if (data == nullptr)
continue;
data = decompress(data.get(), length, CHUNK_DATA_LEN);
if (func(data.get())) {
put(gx, gz, REGION_LAYER_VOXELS, std::move(data), CHUNK_DATA_LEN, true);
}
}
}
}
fs::path WorldFiles::getFolder() const {
return directory;
}
fs::path WorldFiles::getRegionsFolder(int layer) const {
return layers[layer].folder;
}

135
src/files/WorldFiles.h
View File

@ -1,38 +1,24 @@
#ifndef FILES_WORLDFILES_H_
#define FILES_WORLDFILES_H_
#ifndef FILES_WORLD_FILES_H_
#define FILES_WORLD_FILES_H_
#include "WorldRegions.h"
#include "files.h"
#include "../typedefs.h"
#include "../settings.h"
#include "../content/ContentPack.h"
#include "../voxels/Chunk.h"
#include "../util/BufferPool.h"
#include <map>
#include <mutex>
#include <vector>
#include <string>
#include <memory>
#include <unordered_map>
#include <filesystem>
#include <condition_variable>
#include <glm/glm.hpp>
#define GLM_ENABLE_EXPERIMENTAL
#include "glm/gtx/hash.hpp"
inline constexpr uint REGION_HEADER_SIZE = 10;
inline constexpr uint REGION_LAYER_VOXELS = 0;
inline constexpr uint REGION_LAYER_LIGHTS = 1;
inline constexpr uint REGION_LAYER_INVENTORIES = 2;
inline constexpr uint REGION_SIZE_BIT = 5;
inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT));
inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE));
inline constexpr uint REGION_FORMAT_VERSION = 2;
inline constexpr uint WORLD_FORMAT_VERSION = 1;
inline constexpr uint MAX_OPEN_REGION_FILES = 16;
class Player;
class Content;
@ -41,97 +27,19 @@ class World;
namespace fs = std::filesystem;
class illegal_region_format : public std::runtime_error {
public:
illegal_region_format(const std::string& message)
: std::runtime_error(message) {}
};
class WorldRegion {
ubyte** chunksData;
uint32_t* sizes;
bool unsaved = false;
public:
WorldRegion();
~WorldRegion();
void put(uint x, uint z, ubyte* data, uint32_t size);
ubyte* getChunkData(uint x, uint z);
uint getChunkDataSize(uint x, uint z);
void setUnsaved(bool unsaved);
bool isUnsaved() const;
ubyte** getChunks() const;
uint32_t* getSizes() const;
};
struct regfile {
files::rafile file;
int version;
bool inUse = false;
regfile(fs::path filename);
};
using regionsmap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
using regionproc = std::function<bool(ubyte*)>;
struct RegionsLayer {
int layer;
fs::path folder;
regionsmap regions;
std::mutex mutex;
};
class WorldFiles {
fs::path directory;
std::unordered_map<glm::ivec3, std::unique_ptr<regfile>> openRegFiles;
std::mutex regFilesMutex;
std::condition_variable regFilesCv;
RegionsLayer layers[3] {};
WorldRegions regions;
bool generatorTestMode = false;
bool doWriteLights = true;
util::BufferPool<ubyte> bufferPool {
std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2
};
fs::path getRegionFilename(int x, int y) const;
fs::path getWorldFile() const;
fs::path getIndicesFile() const;
fs::path getPacksFile() const;
WorldRegion* getRegion(int x, int z, int layer);
WorldRegion* getOrCreateRegion(int x, int z, int layer);
/// @brief Compress buffer with extrle
/// @param src source buffer
/// @param srclen length of the source buffer
/// @param len (out argument) length of result buffer
/// @return compressed bytes array
std::unique_ptr<ubyte[]> compress(const ubyte* src, size_t srclen, size_t& len);
/// @brief Decompress buffer with extrle
/// @param src compressed buffer
/// @param srclen length of compressed buffer
/// @param dstlen max expected length of source buffer
/// @return decompressed bytes array
std::unique_ptr<ubyte[]> decompress(const ubyte* src, size_t srclen, size_t dstlen);
std::unique_ptr<ubyte[]> readChunkData(int x, int y, uint32_t& length, regfile* file);
void fetchChunks(WorldRegion* region, int x, int y, regfile* file);
void writeWorldInfo(const World* world);
void writeRegions(int layer);
void writeIndices(const ContentIndices* indices);
ubyte* getData(int x, int z, int layer, uint32_t& size);
std::shared_ptr<regfile> getRegFile(glm::ivec3 coord);
void closeRegFile(glm::ivec3 coord);
std::shared_ptr<regfile> useRegFile(glm::ivec3 coord);
std::shared_ptr<regfile> createRegFile(glm::ivec3 coord);
public:
WorldFiles(fs::path directory);
WorldFiles(fs::path directory, const DebugSettings& settings);
@ -140,30 +48,8 @@ public:
fs::path getPlayerFile() const;
void createDirectories();
/// @brief Put all chunk data to regions
void put(Chunk* chunk);
/// @brief Store data in specified region
/// @param x chunk.x
/// @param z chunk.z
/// @param layer regions layer
/// @param data target data
/// @param size data size
/// @param rle compress with ext-RLE
void put(int x, int z, int layer, std::unique_ptr<ubyte[]> data, size_t size, bool rle);
std::unique_ptr<ubyte[]> getChunk(int x, int z);
std::unique_ptr<light_t[]> getLights(int x, int z);
chunk_inventories_map fetchInventories(int x, int z);
bool readWorldInfo(World* world);
/// @brief Write or rewrite region file
/// @param x region X
/// @param z region Z
/// @param layer regions layer
void writeRegion(int x, int y, int layer, WorldRegion* entry);
/// @brief Write all unsaved data to world files
/// @param world target world
/// @param content world content
@ -173,14 +59,15 @@ public:
void removeIndices(const std::vector<std::string>& packs);
void processRegionVoxels(int x, int z, regionproc func);
/// @return world folder
fs::path getFolder() const;
fs::path getRegionsFolder(int layer) const;
static const inline std::string WORLD_FILE = "world.json";
static bool parseRegionFilename(const std::string& name, int& x, int& y);
WorldRegions& getRegions() {
return regions;
}
};
#endif /* FILES_WORLDFILES_H_ */
#endif // FILES_WORLD_FILES_H_

456
src/files/WorldRegions.cpp Normal file
View File

@ -0,0 +1,456 @@
#include "WorldRegions.h"
#include "../coders/rle.h"
#include "../util/data_io.h"
#include "../coders/byte_utils.h"
#include "../maths/voxmaths.h"
#include "../items/Inventory.h"
#include <cstring>
#define REGION_FORMAT_MAGIC ".VOXREG"
regfile::regfile(fs::path filename) : file(filename) {
if (file.length() < REGION_HEADER_SIZE)
throw std::runtime_error("incomplete region file header");
char header[REGION_HEADER_SIZE];
file.read(header, REGION_HEADER_SIZE);
// avoid of use strcmp_s
if (std::string(header, strlen(REGION_FORMAT_MAGIC)) != REGION_FORMAT_MAGIC) {
throw std::runtime_error("invalid region file magic number");
}
version = header[8];
if (uint(version) > REGION_FORMAT_VERSION) {
throw illegal_region_format(
"region format "+std::to_string(version)+" is not supported"
);
}
}
WorldRegion::WorldRegion() {
chunksData = new ubyte*[REGION_CHUNKS_COUNT]{};
sizes = new uint32_t[REGION_CHUNKS_COUNT]{};
}
WorldRegion::~WorldRegion() {
for (uint i = 0; i < REGION_CHUNKS_COUNT; i++) {
delete[] chunksData[i];
}
delete[] sizes;
delete[] chunksData;
}
void WorldRegion::setUnsaved(bool unsaved) {
this->unsaved = unsaved;
}
bool WorldRegion::isUnsaved() const {
return unsaved;
}
ubyte** WorldRegion::getChunks() const {
return chunksData;
}
uint32_t* WorldRegion::getSizes() const {
return sizes;
}
void WorldRegion::put(uint x, uint z, ubyte* data, uint32_t size) {
size_t chunk_index = z * REGION_SIZE + x;
delete[] chunksData[chunk_index];
chunksData[chunk_index] = data;
sizes[chunk_index] = size;
}
ubyte* WorldRegion::getChunkData(uint x, uint z) {
return chunksData[z * REGION_SIZE + x];
}
uint WorldRegion::getChunkDataSize(uint x, uint z) {
return sizes[z * REGION_SIZE + x];
}
WorldRegions::WorldRegions(fs::path directory) : directory(directory) {
for (uint i = 0; i < sizeof(layers)/sizeof(RegionsLayer); i++) {
layers[i].layer = i;
}
layers[REGION_LAYER_VOXELS].folder = directory/fs::path("regions");
layers[REGION_LAYER_LIGHTS].folder = directory/fs::path("lights");
layers[REGION_LAYER_INVENTORIES].folder = directory/fs::path("inventories");
}
WorldRegions::~WorldRegions() {
}
WorldRegion* WorldRegions::getRegion(int x, int z, int layer) {
RegionsLayer& regions = layers[layer];
std::lock_guard lock(regions.mutex);
auto found = regions.regions.find(glm::ivec2(x, z));
if (found == regions.regions.end())
return nullptr;
return found->second.get();
}
WorldRegion* WorldRegions::getOrCreateRegion(int x, int z, int layer) {
RegionsLayer& regions = layers[layer];
WorldRegion* region = getRegion(x, z, layer);
if (region == nullptr) {
std::lock_guard lock(regions.mutex);
region = new WorldRegion();
regions.regions[glm::ivec2(x, z)].reset(region);
}
return region;
}
std::unique_ptr<ubyte[]> WorldRegions::compress(const ubyte* src, size_t srclen, size_t& len) {
auto buffer = bufferPool.get();
ubyte* bytes = buffer.get();
len = extrle::encode(src, srclen, bytes);
auto data = std::make_unique<ubyte[]>(len);
for (size_t i = 0; i < len; i++) {
data[i] = bytes[i];
}
return data;
}
std::unique_ptr<ubyte[]> WorldRegions::decompress(const ubyte* src, size_t srclen, size_t dstlen) {
auto decompressed = std::make_unique<ubyte[]>(dstlen);
extrle::decode(src, srclen, decompressed.get());
return decompressed;
}
inline void calc_reg_coords(
int x, int z, int& regionX, int& regionZ, int& localX, int& localZ
) {
regionX = floordiv(x, REGION_SIZE);
regionZ = floordiv(z, REGION_SIZE);
localX = x - (regionX * REGION_SIZE);
localZ = z - (regionZ * REGION_SIZE);
}
std::unique_ptr<ubyte[]> WorldRegions::readChunkData(
int x,
int z,
uint32_t& length,
regfile* rfile
){
if (generatorTestMode)
return nullptr;
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
int chunkIndex = localZ * REGION_SIZE + localX;
files::rafile& file = rfile->file;
size_t file_size = file.length();
size_t table_offset = file_size - REGION_CHUNKS_COUNT * 4;
uint32_t offset;
file.seekg(table_offset + chunkIndex * 4);
file.read((char*)(&offset), 4);
offset = dataio::read_int32_big((const ubyte*)(&offset), 0);
if (offset == 0){
return nullptr;
}
file.seekg(offset);
file.read((char*)(&offset), 4);
length = dataio::read_int32_big((const ubyte*)(&offset), 0);
auto data = std::make_unique<ubyte[]>(length);
file.read((char*)data.get(), length);
return data;
}
/// @brief Read missing chunks data (null pointers) from region file
void WorldRegions::fetchChunks(WorldRegion* region, int x, int z, regfile* file) {
ubyte** chunks = region->getChunks();
uint32_t* sizes = region->getSizes();
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
int chunk_x = (i % REGION_SIZE) + x * REGION_SIZE;
int chunk_z = (i / REGION_SIZE) + z * REGION_SIZE;
if (chunks[i] == nullptr) {
chunks[i] = readChunkData(chunk_x, chunk_z, sizes[i], file).release();
}
}
}
ubyte* WorldRegions::getData(
int x, int z, int layer,
uint32_t& size
) {
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer);
ubyte* data = region->getChunkData(localX, localZ);
if (data == nullptr) {
auto regfile = getRegFile(glm::ivec3(regionX, regionZ, layer));
if (regfile != nullptr) {
data = readChunkData(x, z, size, regfile.get()).release();
}
if (data != nullptr) {
region->put(localX, localZ, data, size);
}
}
if (data != nullptr) {
size = region->getChunkDataSize(localX, localZ);
return data;
}
return nullptr;
}
std::shared_ptr<regfile> WorldRegions::useRegFile(glm::ivec3 coord) {
return std::shared_ptr<regfile>(openRegFiles[coord].get(), [this](regfile* ptr) {
ptr->inUse = false;
regFilesCv.notify_one();
});
}
void WorldRegions::closeRegFile(glm::ivec3 coord) {
openRegFiles.erase(coord);
regFilesCv.notify_one();
}
// Marks regfile as used and unmarks when shared_ptr dies
std::shared_ptr<regfile> WorldRegions::getRegFile(glm::ivec3 coord) {
{
std::lock_guard lock(regFilesMutex);
const auto found = openRegFiles.find(coord);
if (found != openRegFiles.end()) {
if (found->second->inUse) {
throw std::runtime_error("regfile is currently in use");
}
found->second->inUse = true;
return useRegFile(found->first);
}
}
return createRegFile(coord);
}
std::shared_ptr<regfile> WorldRegions::createRegFile(glm::ivec3 coord) {
fs::path file = layers[coord[2]].folder/getRegionFilename(coord[0], coord[1]);
if (!fs::exists(file)) {
return nullptr;
}
if (openRegFiles.size() == MAX_OPEN_REGION_FILES) {
std::unique_lock lock(regFilesMutex);
while (true) {
bool closed = false;
// FIXME: bad choosing algorithm
for (auto& entry : openRegFiles) {
if (!entry.second->inUse) {
closeRegFile(entry.first);
closed = true;
break;
}
}
if (closed) {
break;
}
// notified when any regfile gets out of use or closed
regFilesCv.wait(lock);
}
openRegFiles[coord] = std::make_unique<regfile>(file);
return useRegFile(coord);
} else {
std::lock_guard lock(regFilesMutex);
openRegFiles[coord] = std::make_unique<regfile>(file);
return useRegFile(coord);
}
}
fs::path WorldRegions::getRegionFilename(int x, int z) const {
return fs::path(std::to_string(x) + "_" + std::to_string(z) + ".bin");
}
void WorldRegions::writeRegion(int x, int z, int layer, WorldRegion* entry){
fs::path filename = layers[layer].folder/getRegionFilename(x, z);
glm::ivec3 regcoord(x, z, layer);
if (auto regfile = getRegFile(regcoord)) {
fetchChunks(entry, x, z, regfile.get());
std::lock_guard lock(regFilesMutex);
closeRegFile(regcoord);
}
char header[REGION_HEADER_SIZE] = REGION_FORMAT_MAGIC;
header[8] = REGION_FORMAT_VERSION;
header[9] = 0; // flags
std::ofstream file(filename, std::ios::out | std::ios::binary);
file.write(header, REGION_HEADER_SIZE);
size_t offset = REGION_HEADER_SIZE;
char intbuf[4]{};
uint offsets[REGION_CHUNKS_COUNT]{};
ubyte** region = entry->getChunks();
uint32_t* sizes = entry->getSizes();
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
ubyte* chunk = region[i];
if (chunk == nullptr){
offsets[i] = 0;
} else {
offsets[i] = offset;
size_t compressedSize = sizes[i];
dataio::write_int32_big(compressedSize, (ubyte*)intbuf, 0);
offset += 4 + compressedSize;
file.write(intbuf, 4);
file.write((const char*)chunk, compressedSize);
}
}
for (size_t i = 0; i < REGION_CHUNKS_COUNT; i++) {
dataio::write_int32_big(offsets[i], (ubyte*)intbuf, 0);
file.write(intbuf, 4);
}
}
void WorldRegions::writeRegions(int layer) {
for (auto& it : layers[layer].regions){
WorldRegion* region = it.second.get();
if (region->getChunks() == nullptr || !region->isUnsaved())
continue;
glm::ivec2 key = it.first;
writeRegion(key[0], key[1], layer, region);
}
}
void WorldRegions::put(int x, int z, int layer, std::unique_ptr<ubyte[]> data, size_t size, bool rle) {
if (rle) {
size_t compressedSize;
auto compressed = compress(data.get(), size, compressedSize);
put(x, z, layer, std::move(compressed), compressedSize, false);
return;
}
int regionX, regionZ, localX, localZ;
calc_reg_coords(x, z, regionX, regionZ, localX, localZ);
WorldRegion* region = getOrCreateRegion(regionX, regionZ, layer);
region->setUnsaved(true);
region->put(localX, localZ, data.release(), size);
}
static std::unique_ptr<ubyte[]> write_inventories(Chunk* chunk, uint& datasize) {
auto& inventories = chunk->inventories;
ByteBuilder builder;
builder.putInt32(inventories.size());
for (auto& entry : inventories) {
builder.putInt32(entry.first);
auto map = entry.second->serialize();
auto bytes = json::to_binary(map.get(), true);
builder.putInt32(bytes.size());
builder.put(bytes.data(), bytes.size());
}
auto datavec = builder.data();
datasize = builder.size();
auto data = std::make_unique<ubyte[]>(datasize);
for (uint i = 0; i < datasize; i++) {
data[i] = datavec[i];
}
return data;
}
/// @brief Store chunk (voxels and lights) in region (existing or new)
void WorldRegions::put(Chunk* chunk){
assert(chunk != nullptr);
int regionX, regionZ, localX, localZ;
calc_reg_coords(chunk->x, chunk->z, regionX, regionZ, localX, localZ);
put(chunk->x, chunk->z, REGION_LAYER_VOXELS,
chunk->encode(), CHUNK_DATA_LEN, true);
// Writing lights cache
if (doWriteLights && chunk->isLighted()) {
put(chunk->x, chunk->z, REGION_LAYER_LIGHTS,
chunk->lightmap.encode(), LIGHTMAP_DATA_LEN, true);
}
// Writing block inventories
if (!chunk->inventories.empty()){
uint datasize;
put(chunk->x, chunk->z, REGION_LAYER_INVENTORIES,
write_inventories(chunk, datasize), datasize, false);
}
}
std::unique_ptr<ubyte[]> WorldRegions::getChunk(int x, int z){
uint32_t size;
auto* data = getData(x, z, REGION_LAYER_VOXELS, size);
if (data == nullptr) {
return nullptr;
}
return decompress(data, size, CHUNK_DATA_LEN);
}
/// @brief Get cached lights for chunk at x,z
/// @return lights data or nullptr
std::unique_ptr<light_t[]> WorldRegions::getLights(int x, int z) {
uint32_t size;
auto* bytes = getData(x, z, REGION_LAYER_LIGHTS, size);
if (bytes == nullptr)
return nullptr;
auto data = decompress(bytes, size, LIGHTMAP_DATA_LEN);
return Lightmap::decode(data.get());
}
chunk_inventories_map WorldRegions::fetchInventories(int x, int z) {
chunk_inventories_map inventories;
uint32_t bytesSize;
const ubyte* data = getData(x, z, REGION_LAYER_INVENTORIES, bytesSize);
if (data == nullptr)
return inventories;
ByteReader reader(data, bytesSize);
int count = reader.getInt32();
for (int i = 0; i < count; i++) {
uint index = reader.getInt32();
uint size = reader.getInt32();
auto map = json::from_binary(reader.pointer(), size);
reader.skip(size);
auto inv = std::make_shared<Inventory>(0, 0);
inv->deserialize(map.get());
inventories[index] = inv;
}
return inventories;
}
void WorldRegions::processRegionVoxels(int x, int z, regionproc func) {
if (getRegion(x, z, REGION_LAYER_VOXELS)) {
throw std::runtime_error("not implemented for in-memory regions");
}
auto regfile = getRegFile(glm::ivec3(x, z, REGION_LAYER_VOXELS));
if (regfile == nullptr) {
throw std::runtime_error("could not open region file");
}
for (uint cz = 0; cz < REGION_SIZE; cz++) {
for (uint cx = 0; cx < REGION_SIZE; cx++) {
int gx = cx + x * REGION_SIZE;
int gz = cz + z * REGION_SIZE;
uint32_t length;
auto data = readChunkData(gx, gz, length, regfile.get());
if (data == nullptr)
continue;
data = decompress(data.get(), length, CHUNK_DATA_LEN);
if (func(data.get())) {
put(gx, gz, REGION_LAYER_VOXELS, std::move(data), CHUNK_DATA_LEN, true);
}
}
}
}
fs::path WorldRegions::getRegionsFolder(int layer) const {
return layers[layer].folder;
}
void WorldRegions::write() {
for (auto& layer : layers) {
fs::create_directories(layer.folder);
writeRegions(layer.layer);
}
}

155
src/files/WorldRegions.h Normal file
View File

@ -0,0 +1,155 @@
#ifndef FILES_WORLD_REGIONS_H_
#define FILES_WORLD_REGIONS_H_
#include "files.h"
#include "../typedefs.h"
#include "../util/BufferPool.h"
#include "../voxels/Chunk.h"
#include <mutex>
#include <memory>
#include <functional>
#include <filesystem>
#include <unordered_map>
#include <condition_variable>
#include <glm/glm.hpp>
#define GLM_ENABLE_EXPERIMENTAL
#include "glm/gtx/hash.hpp"
namespace fs = std::filesystem;
inline constexpr uint REGION_HEADER_SIZE = 10;
inline constexpr uint REGION_LAYER_VOXELS = 0;
inline constexpr uint REGION_LAYER_LIGHTS = 1;
inline constexpr uint REGION_LAYER_INVENTORIES = 2;
inline constexpr uint REGION_SIZE_BIT = 5;
inline constexpr uint REGION_SIZE = (1 << (REGION_SIZE_BIT));
inline constexpr uint REGION_CHUNKS_COUNT = ((REGION_SIZE) * (REGION_SIZE));
inline constexpr uint REGION_FORMAT_VERSION = 2;
inline constexpr uint MAX_OPEN_REGION_FILES = 16;
class illegal_region_format : public std::runtime_error {
public:
illegal_region_format(const std::string& message)
: std::runtime_error(message) {}
};
class WorldRegion {
ubyte** chunksData;
uint32_t* sizes;
bool unsaved = false;
public:
WorldRegion();
~WorldRegion();
void put(uint x, uint z, ubyte* data, uint32_t size);
ubyte* getChunkData(uint x, uint z);
uint getChunkDataSize(uint x, uint z);
void setUnsaved(bool unsaved);
bool isUnsaved() const;
ubyte** getChunks() const;
uint32_t* getSizes() const;
};
struct regfile {
files::rafile file;
int version;
bool inUse = false;
regfile(fs::path filename);
};
using regionsmap = std::unordered_map<glm::ivec2, std::unique_ptr<WorldRegion>>;
using regionproc = std::function<bool(ubyte*)>;
struct RegionsLayer {
int layer;
fs::path folder;
regionsmap regions;
std::mutex mutex;
};
class WorldRegions {
fs::path directory;
std::unordered_map<glm::ivec3, std::unique_ptr<regfile>> openRegFiles;
std::mutex regFilesMutex;
std::condition_variable regFilesCv;
RegionsLayer layers[3] {};
util::BufferPool<ubyte> bufferPool {
std::max(CHUNK_DATA_LEN, LIGHTMAP_DATA_LEN) * 2
};
WorldRegion* getRegion(int x, int z, int layer);
WorldRegion* getOrCreateRegion(int x, int z, int layer);
/// @brief Compress buffer with extrle
/// @param src source buffer
/// @param srclen length of the source buffer
/// @param len (out argument) length of result buffer
/// @return compressed bytes array
std::unique_ptr<ubyte[]> compress(const ubyte* src, size_t srclen, size_t& len);
/// @brief Decompress buffer with extrle
/// @param src compressed buffer
/// @param srclen length of compressed buffer
/// @param dstlen max expected length of source buffer
/// @return decompressed bytes array
std::unique_ptr<ubyte[]> decompress(const ubyte* src, size_t srclen, size_t dstlen);
std::unique_ptr<ubyte[]> readChunkData(int x, int y, uint32_t& length, regfile* file);
void fetchChunks(WorldRegion* region, int x, int y, regfile* file);
ubyte* getData(int x, int z, int layer, uint32_t& size);
std::shared_ptr<regfile> getRegFile(glm::ivec3 coord);
void closeRegFile(glm::ivec3 coord);
std::shared_ptr<regfile> useRegFile(glm::ivec3 coord);
std::shared_ptr<regfile> createRegFile(glm::ivec3 coord);
fs::path getRegionFilename(int x, int y) const;
void writeRegions(int layer);
/// @brief Write or rewrite region file
/// @param x region X
/// @param z region Z
/// @param layer regions layer
void writeRegion(int x, int y, int layer, WorldRegion* entry);
public:
bool generatorTestMode = false;
bool doWriteLights = true;
WorldRegions(fs::path directory);
WorldRegions(const WorldRegions&) = delete;
~WorldRegions();
/// @brief Put all chunk data to regions
void put(Chunk* chunk);
/// @brief Store data in specified region
/// @param x chunk.x
/// @param z chunk.z
/// @param layer regions layer
/// @param data target data
/// @param size data size
/// @param rle compress with ext-RLE
void put(int x, int z, int layer, std::unique_ptr<ubyte[]> data, size_t size, bool rle);
std::unique_ptr<ubyte[]> getChunk(int x, int z);
std::unique_ptr<light_t[]> getLights(int x, int z);
chunk_inventories_map fetchInventories(int x, int z);
void processRegionVoxels(int x, int z, regionproc func);
fs::path getRegionsFolder(int layer) const;
void write();
};
#endif // FILES_WORLD_REGIONS_H_

9
src/voxels/Chunks.cpp
View File

@ -410,6 +410,7 @@ void Chunks::setCenter(int32_t x, int32_t z) {
}
void Chunks::translate(int32_t dx, int32_t dz) {
auto& regions = worldFiles->getRegions();
for (uint i = 0; i < volume; i++){
chunksSecond[i] = nullptr;
}
@ -422,8 +423,9 @@ void Chunks::translate(int32_t dx, int32_t dz) {
continue;
if (nx < 0 || nz < 0 || nx >= int(w) || nz >= int(d)){
events->trigger(EVT_CHUNK_HIDDEN, chunk.get());
if (worldFiles)
worldFiles->put(chunk.get());
if (worldFiles) {
regions.put(chunk.get());
}
chunksCount--;
continue;
}
@ -482,10 +484,11 @@ bool Chunks::putChunk(std::shared_ptr<Chunk> chunk) {
}
void Chunks::saveAndClear(){
auto& regions = worldFiles->getRegions();
for (size_t i = 0; i < volume; i++){
Chunk* chunk = chunks[i].get();
if (chunk) {
worldFiles->put(chunk);
regions.put(chunk);
events->trigger(EVT_CHUNK_HIDDEN, chunk);
}
chunks[i] = nullptr;

8
src/voxels/ChunksStorage.cpp
View File

@ -51,14 +51,14 @@ static void verifyLoadedChunk(ContentIndices* indices, Chunk* chunk) {
std::shared_ptr<Chunk> ChunksStorage::create(int x, int z) {
World* world = level->getWorld();
WorldFiles* wfile = world->wfile.get();
auto& regions = world->wfile.get()->getRegions();
auto chunk = std::make_shared<Chunk>(x, z);
store(chunk);
auto data = wfile->getChunk(chunk->x, chunk->z);
auto data = regions.getChunk(chunk->x, chunk->z);
if (data) {
chunk->decode(data.get());
auto invs = wfile->fetchInventories(chunk->x, chunk->z);
auto invs = regions.fetchInventories(chunk->x, chunk->z);
chunk->setBlockInventories(std::move(invs));
chunk->setLoaded(true);
for(auto& entry : chunk->inventories) {
@ -67,7 +67,7 @@ std::shared_ptr<Chunk> ChunksStorage::create(int x, int z) {
verifyLoadedChunk(level->content->getIndices(), chunk.get());
}
auto lights = wfile->getLights(chunk->x, chunk->z);
auto lights = regions.getLights(chunk->x, chunk->z);
if (lights) {
chunk->lightmap.set(lights.get());
chunk->setLoadedLights(true);

4
src/world/World.cpp
View File

@ -49,8 +49,8 @@ void World::updateTimers(float delta) {
void World::write(Level* level) {
const Content* content = level->content;
Chunks* chunks = level->chunks.get();
auto& regions = wfile->getRegions();
for (size_t i = 0; i < chunks->volume; i++) {
auto chunk = chunks->chunks[i];
@ -60,7 +60,7 @@ void World::write(Level* level) {
settings.debug.doWriteLights;
if (!chunk->isUnsaved() && !lightsUnsaved)
continue;
wfile->put(chunk.get());
regions.put(chunk.get());
}
wfile->write(this, content);