GPUBufferObject.hpp

Go to the documentation of this file.

//
//  CeresEngine - A game development framework
//
//  Created by Rogiel Sulzbach.
//  Copyright (c) 2018-2022 Rogiel Sulzbach. All rights reserved.
//
 
#pragma once
 
#include "GPUBuffer.hpp"
#include "GPUCommandBuffer.hpp"
#include "GPUDevice.hpp"
#include "GPUResourceSet.hpp"
 
#include "CeresEngine/DataTypes.hpp"
#include "CeresEngine/Macros.hpp"
 
#include "CeresEngine/Foundation/Iterator.hpp"
#include "CeresEngine/Foundation/SmartPtr.hpp"
#include "CeresEngine/Foundation/Container/MemoryView.hpp"
#include "CeresEngine/Foundation/IO/Stream.hpp"
 
#include <cstring>
 
namespace CeresEngine {
 
    class GPUBufferObject {
    protected:
        GPUBufferPtr mBuffer = nullptr;
 
        GPUBufferSize mOffset = 0;
 
        GPUBufferSize mLength = GPUBuffer::whole;
 
    public:
        GPUBufferObject() = default;
 
        explicit GPUBufferObject(GPUBufferPtr buffer, GPUBufferSize offset = 0, GPUBufferSize length = GPUBuffer::whole);
 
        explicit GPUBufferObject(GPUBuffer& buffer, const GPUBufferSize offset = 0, const GPUBufferSize length = GPUBuffer::whole)
                : GPUBufferObject(&buffer, offset, length) {}
 
        explicit GPUBufferObject(GPUDevice& device, const GPUBufferDescriptor& descriptor);
 
        GPUBufferObject(std::nullptr_t) {} // NOLINT
 
        GPUBufferObject& operator=(std::nullptr_t) {
            mBuffer = nullptr;
            mOffset = 0;
            return *this;
        }
 
        GPUBufferObject(const GPUBufferObject&) = default;
 
        GPUBufferObject& operator=(const GPUBufferObject&) = default;
 
        GPUBufferObject(GPUBufferObject&&) = default;
 
        GPUBufferObject& operator=(GPUBufferObject&&) = default;
 
        ~GPUBufferObject();
 
    public:
        void flush(GPUBufferSize offset = 0, GPUBufferSize size = GPUBuffer::whole);
 
        void invalidate(GPUBufferSize offset = 0, GPUBufferSize size = GPUBuffer::whole);
 
    public:
        [[nodiscard]] GPUBufferPtr getBuffer() const noexcept { return mBuffer; }
 
        [[nodiscard]] GPUBufferSize getOffset() const noexcept { return mOffset; }
 
        [[nodiscard]] GPUBufferSize getLength() const noexcept { return mLength == GPUBuffer::whole ? mBuffer->getSize() - mOffset : mLength; }
 
    public:
        explicit operator bool() const noexcept { return mBuffer != nullptr; }
 
        [[nodiscard]] friend bool operator==(const GPUBufferObject& lhs, const GPUBufferObject& rhs) noexcept {
            return lhs.getBuffer() == rhs.getBuffer() && lhs.getOffset() == rhs.getOffset() && lhs.getLength() == rhs.getLength();
        }
        [[nodiscard]] friend bool operator==(const GPUBufferObject& lhs, std::nullptr_t) noexcept { return lhs.getBuffer() == nullptr; }
        [[nodiscard]] friend bool operator==(std::nullptr_t, const GPUBufferObject& rhs) noexcept { return rhs.getBuffer() == nullptr; }
 
        [[nodiscard]] friend bool operator!=(const GPUBufferObject& lhs, const GPUBufferObject& rhs) noexcept {
            return lhs.getBuffer() != rhs.getBuffer() && lhs.getOffset() != rhs.getOffset() && lhs.getLength() != rhs.getLength();
        }
        [[nodiscard]] friend bool operator!=(const GPUBufferObject& lhs, std::nullptr_t) noexcept { return lhs.getBuffer() != nullptr; }
        [[nodiscard]] friend bool operator!=(std::nullptr_t, const GPUBufferObject& rhs) noexcept { return rhs.getBuffer() != nullptr; }
    };
 
    class GPUVertexBuffer : public GPUBufferObject {
    public:
        GPUVertexBuffer() = default;
 
        explicit GPUVertexBuffer(GPUBufferPtr buffer, GPUBufferSize offset = 0, GPUBufferSize length = GPUBuffer::whole);
 
        explicit GPUVertexBuffer(GPUBuffer& buffer, const GPUBufferSize offset = 0, const GPUBufferSize length = GPUBuffer::whole)
                : GPUVertexBuffer(&buffer, offset, length) {}
 
        explicit GPUVertexBuffer(GPUDevice& device, GPUBufferSize size, const GPUMemoryProperties& memoryProperty = GPUMemoryProperty::DeviceLocal,
            String name = "");
 
        GPUVertexBuffer(std::nullptr_t) {} // NOLINT
 
        GPUVertexBuffer(const GPUVertexBuffer&) = default;
 
        GPUVertexBuffer& operator=(const GPUVertexBuffer&) = default;
 
        GPUVertexBuffer(GPUVertexBuffer&&) = default;
 
        GPUVertexBuffer& operator=(GPUVertexBuffer&&) = default;
 
        ~GPUVertexBuffer();
    };
 
    class GPUIndexBuffer : public GPUBufferObject {
        IndexType mIndexType = IndexType::UInt32;
 
    public:
        GPUIndexBuffer() = default;
 
        explicit GPUIndexBuffer(GPUBufferPtr buffer, GPUBufferSize offset = 0, IndexType indexType = IndexType::UInt32, GPUBufferSize length = GPUBuffer::whole);
 
        explicit GPUIndexBuffer(GPUBuffer& buffer, const GPUBufferSize offset = 0, const IndexType indexType = IndexType::UInt32, const GPUBufferSize length = GPUBuffer::whole)
                : GPUIndexBuffer(&buffer, offset, indexType, length) {}
 
        explicit GPUIndexBuffer(GPUDevice& device, GPUBufferSize size, IndexType indexType = IndexType::UInt32,
            const GPUMemoryProperties& memoryProperty = GPUMemoryProperty::DeviceLocal, String name = "");
 
        GPUIndexBuffer(std::nullptr_t) {} // NOLINT
 
        GPUIndexBuffer(const GPUIndexBuffer&) = default;
 
        GPUIndexBuffer& operator=(const GPUIndexBuffer&) = default;
 
        GPUIndexBuffer(GPUIndexBuffer&&) = default;
 
        GPUIndexBuffer& operator=(GPUIndexBuffer&&) = default;
 
        ~GPUIndexBuffer();
 
    public:
        [[nodiscard]] IndexType getIndexType() const noexcept { return mIndexType; }
    };
 
    class GPUUniformBuffer : public GPUBufferObject {
    public:
        GPUUniformBuffer() = default;
 
        explicit GPUUniformBuffer(GPUBufferPtr buffer, GPUBufferSize offset = 0, GPUBufferSize length = GPUBuffer::whole);
 
        explicit GPUUniformBuffer(GPUBuffer& buffer, const GPUBufferSize offset = 0, const GPUBufferSize length = GPUBuffer::whole)
                : GPUUniformBuffer(&buffer, offset, length) {}
 
        explicit GPUUniformBuffer(GPUDevice& device, GPUBufferSize length, const GPUMemoryProperties& memoryProperty = GPUMemoryProperty::DeviceLocal,
            String name = "");
 
        explicit GPUUniformBuffer(const GPUStorageBuffer& storageBuffer);
 
        GPUUniformBuffer(std::nullptr_t) {} // NOLINT
 
        GPUUniformBuffer(const GPUUniformBuffer&) = default;
 
        GPUUniformBuffer& operator=(const GPUUniformBuffer&) = default;
 
        GPUUniformBuffer(GPUUniformBuffer&&) = default;
 
        GPUUniformBuffer& operator=(GPUUniformBuffer&&) = default;
 
        ~GPUUniformBuffer();
    };
 
    class GPUStorageBuffer : public GPUBufferObject {
    public:
        GPUStorageBuffer() = default;
 
        explicit GPUStorageBuffer(GPUBufferPtr buffer, GPUBufferSize offset = 0, GPUBufferSize length = GPUBuffer::whole);
 
        explicit GPUStorageBuffer(GPUBuffer& buffer, const GPUBufferSize offset = 0, const GPUBufferSize length = GPUBuffer::whole)
                : GPUStorageBuffer(&buffer, offset, length) {}
 
        explicit GPUStorageBuffer(GPUDevice& device, GPUBufferSize size, const GPUMemoryProperties& memoryProperty = GPUMemoryProperty::DeviceLocal,
            String name = "");
 
        explicit GPUStorageBuffer(const GPUUniformBuffer& uniformBuffer);
 
        GPUStorageBuffer(std::nullptr_t) {} // NOLINT
 
        GPUStorageBuffer(const GPUStorageBuffer&) = default;
 
        GPUStorageBuffer& operator=(const GPUStorageBuffer&) = default;
 
        GPUStorageBuffer(GPUStorageBuffer&&) = default;
 
        GPUStorageBuffer& operator=(GPUStorageBuffer&&) = default;
 
        ~GPUStorageBuffer();
    };
 
    class GPUIndirectBuffer : public GPUBufferObject {
    public:
        GPUIndirectBuffer() = default;
 
        explicit GPUIndirectBuffer(GPUBufferPtr buffer, GPUBufferSize offset = 0, GPUBufferSize length = GPUBuffer::whole);
 
        explicit GPUIndirectBuffer(GPUBuffer& buffer, const GPUBufferSize offset = 0, const GPUBufferSize length = GPUBuffer::whole)
                : GPUIndirectBuffer(&buffer, offset, length) {}
 
        explicit GPUIndirectBuffer(GPUDevice& device, GPUBufferSize size, const GPUMemoryProperties& memoryProperty = GPUMemoryProperty::DeviceLocal,
            String name = "");
 
        GPUIndirectBuffer(std::nullptr_t) {} // NOLINT
 
        GPUIndirectBuffer(const GPUIndirectBuffer&) = default;
 
        GPUIndirectBuffer& operator=(const GPUIndirectBuffer&) = default;
 
        GPUIndirectBuffer(GPUIndirectBuffer&&) = default;
 
        GPUIndirectBuffer& operator=(GPUIndirectBuffer&&) = default;
 
        ~GPUIndirectBuffer();
    };
 
    class GPUStagingBuffer : public GPUBufferObject {
    public:
        GPUStagingBuffer() = default;
 
        explicit GPUStagingBuffer(GPUBufferPtr buffer, GPUBufferSize offset = 0, GPUBufferSize length = GPUBuffer::whole);
 
        explicit GPUStagingBuffer(GPUBuffer& buffer, const GPUBufferSize offset = 0, const GPUBufferSize length = GPUBuffer::whole)
                : GPUStagingBuffer(&buffer, offset, length) {}
 
        explicit GPUStagingBuffer(GPUDevice& device, GPUBufferSize size, const GPUMemoryProperties& memoryProperty = GPUMemoryProperty::HostVisible,
            String name = "");
 
        explicit GPUStagingBuffer(GPUDevice& device, const void* data, GPUBufferSize size,
            const GPUMemoryProperties& memoryProperty = GPUMemoryProperty::HostVisible, String name = "");
 
        explicit GPUStagingBuffer(GPUDevice& device, const GPUImageDescriptor& imageDescriptor,
            const GPUMemoryProperties& memoryProperty = GPUMemoryProperty::HostVisible, String name = "");
 
        GPUStagingBuffer(std::nullptr_t) {} // NOLINT
 
        GPUStagingBuffer(const GPUStagingBuffer&) = default;
 
        GPUStagingBuffer& operator=(const GPUStagingBuffer&) = default;
 
        GPUStagingBuffer(GPUStagingBuffer&&) = default;
 
        GPUStagingBuffer& operator=(GPUStagingBuffer&&) = default;
 
        ~GPUStagingBuffer();
 
    public:
        void copy(const void* ptr, UInt64 size);
        
        void copy(InputStream stream);
 
        void fill(uint8_t byte = 0);
 
        void upload(GPUBuffer& dstBuffer, UInt64 dstOffset = 0, UInt64 srcOffset = 0, UInt64 size = ~0u);
 
        void upload(GPUCommandBuffer& commandBuffer, GPUBuffer& dstBuffer, UInt64 dstOffset = 0, UInt64 srcOffset = 0, UInt64 size = ~0u);
 
        void download(GPUBuffer& srcBuffer, UInt64 srcOffset = 0, UInt64 dstOffset = 0, UInt64 size = ~0u);
 
        void download(GPUCommandBuffer& commandBuffer, GPUBuffer& srcBuffer, UInt64 srcOffset = 0, UInt64 dstOffset = 0, UInt64 size = ~0u);
 
        void upload(GPUImage& dstImage, const GPUImageRegion& dstRegion = {}, UInt64 srcOffset = 0, bool generateMips = true, UInt32 rowStride = 0,
            UInt32 layerStride = 0);
 
        void upload(GPUCommandBuffer& commandBuffer, GPUImage& dstImage, const GPUImageRegion& dstRegion = {}, UInt64 srcOffset = 0, bool generateMips = true,
            UInt32 rowStride = 0, UInt32 layerStride = 0);
 
        void download(GPUImage& srcImage, const GPUImageRegion& srcImageRegion = {}, UInt64 dstOffset = 0, UInt32 rowStride = 0, UInt32 layerStride = 0);
 
        void download(GPUCommandBuffer& commandBuffer, GPUImage& srcImage, const GPUImageRegion& srcImageRegion = {}, UInt64 dstOffset = 0,
            UInt32 rowStride = 0, UInt32 layerStride = 0);
    };
 
    class GPUBindlessBuffer : public GPUBufferObject {
    public:
        GPUBindlessBuffer() = default;
 
        explicit GPUBindlessBuffer(GPUBufferPtr buffer, GPUBufferSize offset = 0, GPUBufferSize length = GPUBuffer::whole);
 
        explicit GPUBindlessBuffer(GPUBuffer& buffer, const GPUBufferSize offset = 0, const GPUBufferSize length = GPUBuffer::whole)
                : GPUBindlessBuffer(&buffer, offset, length) {}
 
        explicit GPUBindlessBuffer(GPUDevice& device, GPUBufferSize size, const GPUMemoryProperties& memoryProperty = GPUMemoryProperty::DeviceLocal,
            String name = "");
 
        GPUBindlessBuffer(std::nullptr_t) {} // NOLINT
 
        GPUBindlessBuffer(const GPUBindlessBuffer&) = default;
 
        GPUBindlessBuffer& operator=(const GPUBindlessBuffer&) = default;
 
        GPUBindlessBuffer(GPUBindlessBuffer&&) = default;
 
        GPUBindlessBuffer& operator=(GPUBindlessBuffer&&) = default;
 
        ~GPUBindlessBuffer();
    };
 
    // ---------------------------------------------------------------------------------------------
    // ---------------------------------------------------------------------------------------------
 
    template<typename P, typename T, std::size_t BaseAlignment = 1> class GPUDynamicBuffer final : public P {
        T* mData = nullptr;
 
    public:
        GPUDynamicBuffer() = default;
 
        explicit GPUDynamicBuffer(GPUBufferPtr buffer, GPUBufferSize offset = 0) : P(std::move(buffer), offset, sizeof(T)) {
            mData = reinterpret_cast<T*>(reinterpret_cast<UInt8*>(P::getBuffer()->map()) + P::getOffset());
        }
 
        template<typename... Args>
        explicit GPUDynamicBuffer(GPUDevice& device, String name = "", Args&&... args)
                : P(device, sizeof(T), args..., GPUMemoryProperty::HostVisible | GPUMemoryProperty::HostCached, name) {
            mData = reinterpret_cast<T*>(reinterpret_cast<UInt8*>(P::getBuffer()->map()) + P::getOffset());
        }
 
        explicit GPUDynamicBuffer(P&& rawBuffer) : P(std::forward<P>(rawBuffer)) {
            mData = reinterpret_cast<T*>(reinterpret_cast<UInt8*>(P::getBuffer()->map()) + P::getOffset());
        }
 
        GPUDynamicBuffer(const GPUDynamicBuffer& value) = default;
        GPUDynamicBuffer(GPUDynamicBuffer&& value) noexcept = default;
 
        ~GPUDynamicBuffer() {
            if(const GPUBufferPtr& buffer = P::getBuffer()) {
                buffer->unmap();
            }
        }
 
        GPUDynamicBuffer& operator=(const GPUDynamicBuffer& value) = default;
        GPUDynamicBuffer& operator=(GPUDynamicBuffer&& value) noexcept = default;
 
        GPUDynamicBuffer& operator=(const T& value) {
            *mData = value;
            return *this;
        }
 
    public:
        [[nodiscard]] T* get() noexcept { return mData; }
 
        [[nodiscard]] const T* get() const noexcept { return mData; }
 
        void update(const T& data) { memcpy(mData, &data, sizeof(T)); }
 
        void commit() { P::flush(); }
 
    public:
        [[nodiscard]] T* operator->() noexcept { return mData; }
 
        [[nodiscard]] const T* operator->() const noexcept { return mData; }
 
        [[nodiscard]] T& operator*() noexcept { return *mData; }
 
        [[nodiscard]] const T& operator*() const noexcept { return *mData; }
    };
 
    template<typename P, typename T, std::size_t BaseAlignment> class GPUDynamicBuffer<P, T[], BaseAlignment> : public P { // NOLINT
    public:
        static constexpr GPUBufferSize byteStride = ((sizeof(T) + BaseAlignment - 1) / BaseAlignment) * BaseAlignment;
 
        static constexpr bool isStrided = byteStride == sizeof(T);
 
        using MemoryViewType = std::conditional_t<isStrided, StridedMemoryView<T>, MemoryView<T>>;
 
        using IteratorType = typename MemoryViewType::Iterator;
 
        using ConstIteratorType = typename MemoryViewType::ConstIterator;
 
    private:
        MemoryViewType mData;
 
    public:
        GPUDynamicBuffer() = default;
 
        explicit GPUDynamicBuffer(GPUBufferPtr buffer, GPUBufferSize offset = 0) : P(std::move(buffer), offset) {
            const GPUBufferData ptr = P::getBuffer()->map() + offset;
            if constexpr(isStrided) {
                mData = make_strided_memory_view(static_cast<T*>(ptr), P::getLength() / byteStride, byteStride);
            } else {
                mData = make_memory_view(static_cast<T*>(ptr), P::getLength() / byteStride);
            }
        }
 
        template<typename... Args>
        explicit GPUDynamicBuffer(GPUDevice& device, const size_t size, String name, Args&&... args)
                : P(device, size * byteStride, args..., GPUMemoryProperty::HostVisible | GPUMemoryProperty::HostCached, name) {
            const GPUBufferData ptr = P::getBuffer()->map();
            if constexpr(isStrided) {
                mData = make_strided_memory_view(static_cast<T*>(ptr), P::getLength() / byteStride, byteStride);
            } else {
                mData = make_memory_view(static_cast<T*>(ptr), P::getLength() / byteStride);
            }
        }
 
        ~GPUDynamicBuffer() {
            if(const GPUBufferPtr& buffer = P::getBuffer()) {
                buffer->unmap();
            }
        }
 
    public:
        [[nodiscard]] T* data() noexcept { return mData.data(); }
        [[nodiscard]] const T* data() const noexcept { return mData.data(); }
 
        void update(const T* data, const size_t length) { update(data, 0, length); }
        void update(const T* data, const size_t offset, const size_t length) {
            for(const size_t index : range(length)) {
                mData[offset + index] = data[index];
            }
        }
 
        [[nodiscard]] GPUDynamicBuffer<P, T> get(const size_t index) const { return GPUDynamicBuffer<P, T>{P::getBuffer(), P::getOffset() + align(index)}; }
        void set(size_t i, const T& value) noexcept { mData[i] = value; }
 
        [[nodiscard]] std::size_t size() const noexcept { return mData.size(); }
        [[nodiscard]] size_t stride() const noexcept requires(isStrided) { return mData.stride(); }
 
        [[nodiscard]] P slice(const size_t index) const { return P{P::getBuffer(), P::getOffset() + align(index), sizeof(T)}; }
 
    public:
        [[nodiscard]] IteratorType begin() noexcept { return mData.begin(); }
        [[nodiscard]] ConstIteratorType begin() const noexcept { return mData.begin(); }
 
        [[nodiscard]] IteratorType end() noexcept { return mData.end(); }
        [[nodiscard]] ConstIteratorType end() const noexcept { return mData.end(); }
 
    public:
        [[nodiscard]] GPUDynamicBuffer<P, T> operator[](const size_t index) const noexcept { return GPUDynamicBuffer<P, T>(slice(index)); }
 
    private:
        [[nodiscard]] size_t align(const size_t index) const noexcept { return byteStride * index; }
    };
 
    template<typename P, typename T> class GPUDynamicBuffer<P, T[], 0> : public P { // NOLINT
    public:
        static constexpr bool isStrided = true;
 
        using MemoryViewType = StridedMemoryView<T>;
 
        using IteratorType = typename MemoryViewType::Iterator;
 
        using ConstIteratorType = typename MemoryViewType::ConstIterator;
 
    private:
        MemoryViewType mData;
 
    public:
        GPUDynamicBuffer() = default;
 
        explicit GPUDynamicBuffer(GPUBufferPtr buffer, const GPUBufferSize baseAlignment = sizeof(T), GPUBufferSize offset = 0) : P(std::move(buffer), offset) {
            const GPUBufferSize stride = ((sizeof(T) + baseAlignment - 1) / baseAlignment) * baseAlignment;
            const GPUBufferData ptr = P::getBuffer()->map() + offset;
            mData = make_strided_memory_view(static_cast<T*>(ptr), P::getLength() / stride, stride);
        }
 
        template<typename... Args>
        explicit GPUDynamicBuffer(GPUDevice& device, const size_t size, const size_t baseAlignment, String name, Args&&... args)
                : P(device, size * ((sizeof(T) + baseAlignment - 1) / baseAlignment) * baseAlignment, args...,
                      GPUMemoryProperty::HostVisible | GPUMemoryProperty::HostCached, name) {
            const GPUBufferSize stride = ((sizeof(T) + baseAlignment - 1) / baseAlignment) * baseAlignment;
            const GPUBufferData ptr = P::getBuffer()->map();
            mData = make_strided_memory_view(static_cast<T*>(ptr), P::getLength() / stride, stride);
        }
 
        ~GPUDynamicBuffer() {
            if(const GPUBufferPtr& buffer = P::getBuffer()) {
                buffer->unmap();
            }
        }
 
    public:
        [[nodiscard]] T* data() noexcept { return mData.data(); }
        [[nodiscard]] const T* data() const noexcept { return mData.data(); }
 
        void update(const T* data, const size_t length) { update(data, 0, length); }
        void update(const T* data, const size_t offset, const size_t length) {
            for(const size_t index : range(length)) {
                mData[offset + index] = data[index];
            }
        }
 
        [[nodiscard]] GPUDynamicBuffer<P, T> get(const size_t index) const { return GPUDynamicBuffer<P, T>{P::getBuffer(), P::getOffset() + align(index)}; }
        void set(size_t i, const T& value) noexcept { mData[i] = value; }
 
        [[nodiscard]] std::size_t size() const noexcept { return mData.size(); }
        [[nodiscard]] size_t stride() const noexcept { return mData.stride(); }
 
        [[nodiscard]] P slice(const size_t index) const { return P{P::getBuffer(), P::getOffset() + align(index), sizeof(T)}; }
 
    public:
        [[nodiscard]] IteratorType begin() noexcept { return mData.begin(); }
        [[nodiscard]] ConstIteratorType begin() const noexcept { return mData.begin(); }
 
        [[nodiscard]] IteratorType end() noexcept { return mData.end(); }
        [[nodiscard]] ConstIteratorType end() const noexcept { return mData.end(); }
 
    public:
        [[nodiscard]] GPUDynamicBuffer<P, T> operator[](const size_t index) const noexcept { return GPUDynamicBuffer<P, T>(slice(index)); }
 
    private:
        [[nodiscard]] size_t align(const size_t index) const noexcept { return stride() * index; }
    };
 
    template<typename P, typename T, size_t N, std::size_t BaseAlignment>
    class GPUDynamicBuffer<P, T[N], BaseAlignment> : public GPUDynamicBuffer<P, T[], BaseAlignment> { // NOLINT
        using super = GPUDynamicBuffer<P, T[], BaseAlignment>;
 
    public:
        GPUDynamicBuffer() = default;
 
        explicit GPUDynamicBuffer(GPUBufferPtr buffer, GPUBufferSize offset = 0) : super(buffer, offset) { CE_ASSERT(super::size() == N); }
 
        template<typename... Args>
        explicit GPUDynamicBuffer(GPUDevice& device, String name = "", Args&&... args) : super(device, N, name, std::forward<Args>(args)...) {
            CE_ASSERT(super::size() == N);
        }
 
        ~GPUDynamicBuffer() {
            if(const GPUBufferPtr& buffer = P::getBuffer()) {
                buffer->unmap();
            }
        }
    };
 
    template<typename T> using GPUDynamicVertexBuffer = GPUDynamicBuffer<GPUVertexBuffer, T[]>; // NOLINT
 
    template<typename T> using GPUDynamicStridedVertexBuffer = GPUDynamicBuffer<GPUVertexBuffer, T[], 0>; // NOLINT
 
    template<typename T> using GPUDynamicIndexBuffer = GPUDynamicBuffer<GPUIndexBuffer, T[]>; // NOLINT
 
    template<typename T> using GPUDynamicStridedIndexBuffer = GPUDynamicBuffer<GPUIndexBuffer, T[], 0>; // NOLINT
 
    template<typename T> using GPUDynamicUniformBuffer = GPUDynamicBuffer<GPUUniformBuffer, T>; // NOLINT
 
    template<typename T, std::size_t BaseAlignment = sizeof(T)>
    using GPUDynamicUniformBufferArray = GPUDynamicBuffer<GPUUniformBuffer, T[], BaseAlignment>; // NOLINT
 
    template<typename T> using GPUDynamicStridedUniformBufferArray = GPUDynamicUniformBufferArray<T, 0>; // NOLINT
 
    template<typename T> using GPUDynamicStorageBuffer = GPUDynamicBuffer<GPUStorageBuffer, T>; // NOLINT
 
    template<typename T, std::size_t BaseAlignment = sizeof(T)>
    using GPUDynamicStorageBufferArray = GPUDynamicBuffer<GPUStorageBuffer, T[], BaseAlignment>; // NOLINT
 
    template<typename T> using GPUDynamicStridedStorageBufferArray = GPUDynamicStorageBufferArray<T, 0>; // NOLINT
 
    template<typename T> using GPUDynamicIndirectBuffer = GPUDynamicBuffer<GPUIndirectBuffer, T>; // NOLINT
 
    template<typename T, std::size_t BaseAlignment = sizeof(T)>
    using GPUDynamicIndirectBufferArray = GPUDynamicBuffer<GPUIndirectBuffer, T[], BaseAlignment>; // NOLINT
 
    template<typename T> using GPUDynamicStridedIndirectBufferArray = GPUDynamicIndirectBufferArray<T, 0>; // NOLINT
 
    template<typename T> using GPUDynamicBindlessBuffer = GPUDynamicBuffer<GPUBindlessBuffer, T>; // NOLINT
 
    template<typename T, std::size_t BaseAlignment = sizeof(T)>
    using GPUDynamicBindlessBufferArray = GPUDynamicBuffer<GPUBindlessBuffer, T[], BaseAlignment>; // NOLINT
 
    template<typename T> using GPUDynamicStridedBindlessBufferArray = GPUDynamicBindlessBufferArray<T, 0>; // NOLINT
 
    // ---------------------------------------------------------------------------------------------
    // ---------------------------------------------------------------------------------------------
 
    template<template<typename> class B> class GPUBufferPool final {
    public:
        using RawBuffer = B<UInt8[]>; // NOLINT
 
    private:
        GPUDevicePtr device = nullptr;
 
        Vector<RawBuffer> buffers{};
 
        GPUBufferSize next = 0;
 
        GPUBufferSize length = 0;
 
    private: // Parameters
        const GPUBufferSize baseAlignment = 256; // TODO query this from the device
 
        const GPUBufferSize baseAllocationSize = 4 * 1024 * 1024;
 
    public:
        explicit GPUBufferPool(GPUDevice& device) : device(&device) {}
 
        explicit GPUBufferPool(GPUDevice& device, const GPUBufferSize baseAllocationSize) : device(&device), baseAllocationSize(baseAllocationSize) {}
 
    public:
        void reserve(const GPUBufferSize length) { allocateBuffers(length); }
 
        template<typename T> [[nodiscard]] B<T> allocate() {
            const size_t alignment = ((sizeof(T) + baseAlignment - 1) / baseAlignment) * baseAlignment;
 
            // If the required alignment is very large (i.e. larger than half
            // the allocation size) we skip any pooling and allocate an entire
            // new buffer for this allocation.
            //
            if(alignment > baseAllocationSize / 2) {
                return B<T>{*device};
            }
 
            if(length < next + alignment) {
                reserve(length * 2 + 1);
            }
 
            GPUBufferSize bufferIndex = next / baseAllocationSize;
            const GPUBufferSize bufferOffset = next - bufferIndex * baseAllocationSize;
            next += alignment;
 
            CE_ASSERT(buffers.size() > bufferIndex);
            RawBuffer& buffer = buffers[bufferIndex];
            return B<T>(buffer.getBuffer(), buffer.getOffset() + static_cast<GPUBufferSize>(bufferOffset));
        }
 
        void reset() noexcept { next = 0; }
 
        [[nodiscard]] bool empty() const noexcept { return next == 0; }
 
    public:
        void flush(const GPUBufferSize offset = 0, const GPUBufferSize size = GPUBuffer::whole) {
            const GPUBufferSize startBufferIndex = offset / baseAllocationSize;
            const GPUBufferSize endBufferIndex = std::min<GPUBufferSize>(((offset + size) / baseAllocationSize) + 1, buffers.size());
 
            for(GPUBufferSize bufferIndex = startBufferIndex; bufferIndex < endBufferIndex; bufferIndex++) {
                RawBuffer& buffer = buffers[bufferIndex];
                buffer.flush();
            }
        }
 
        void invalidate(const GPUBufferSize offset = 0, const GPUBufferSize size = GPUBuffer::whole) {
            const GPUBufferSize startBufferIndex = (offset / baseAllocationSize);
            const GPUBufferSize endBufferIndex = std::min(((offset + size) / baseAllocationSize) + 1, buffers.size());
 
            for(GPUBufferSize bufferIndex = startBufferIndex; bufferIndex < endBufferIndex; bufferIndex++) {
                RawBuffer& buffer = buffers[bufferIndex];
                buffer.invalidate();
            }
        }
 
    private:
        void allocateBuffers(const size_t newLength) {
            const GPUBufferSize numBuffers = (newLength + baseAllocationSize - 1) / baseAllocationSize;
            const GPUBufferSize currentNumBuffers = buffers.size();
            if(currentNumBuffers >= numBuffers) {
                return;
            }
 
            const GPUBufferSize numBuffersToAllocate = numBuffers - currentNumBuffers;
            for(GPUBufferSize i = 0; i < numBuffersToAllocate; i++) {
                buffers.emplace_back(*device, baseAllocationSize, "");
            }
            length = baseAllocationSize * buffers.size();
        }
    };
 
    using GPUUniformBufferPool = GPUBufferPool<GPUDynamicUniformBuffer>;
 
    using GPUStorageBufferPool = GPUBufferPool<GPUDynamicStorageBuffer>;
 
    using GPUBindlessBufferPool = GPUBufferPool<GPUDynamicBindlessBuffer>;
 
    template<typename T, template<typename> class B = GPUDynamicUniformBufferArray> class TGPUBufferPool final {
        static_assert(std::is_trivially_copyable_v<T>);
 
    public:
        using BufferType = B<UInt8>;
 
    private:
        GPUDevicePtr device;
 
        Vector<BufferType> buffers;
 
    private: // Parameters
        static constexpr size_t alignment = ((sizeof(T) + 256 - 1) / 256) * 256;
 
        const size_t baseAllocationSize = 4 * 1024 * 1024;
 
    private: // State
        size_t mLength = 0;
 
    public:
        explicit TGPUBufferPool(GPUDevice& device) : device(&device) {}
 
    public:
        void reserve(const size_t length) { allocateBuffers(length * alignment); }
 
        [[nodiscard]] T& operator[](const size_t index) {
            if(mLength <= index) {
                reserve(index * 2 + 1);
            }
 
            const size_t offset = alignment * index;
            size_t bufferIndex = offset / baseAllocationSize;
            size_t bufferOffset = offset - bufferIndex * baseAllocationSize;
 
            BufferType& buffer = buffers[bufferIndex];
            return reinterpret_cast<T&>(buffer[bufferOffset]);
        }
 
    public:
        void flush(const GPUBufferSize offset = 0, const GPUBufferSize size = GPUBuffer::whole) {
            const size_t startBufferIndex = offset / baseAllocationSize;
            const size_t endBufferIndex = std::min(((offset + size) / baseAllocationSize) + 1, buffers.size());
 
            for(size_t bufferIndex = startBufferIndex; bufferIndex < endBufferIndex; bufferIndex++) {
                BufferType& buffer = buffers[bufferIndex];
                buffer.flush();
            }
        }
 
        void invalidate(const GPUBufferSize offset = 0, const GPUBufferSize size = GPUBuffer::whole) {
            const size_t startBufferIndex = offset / baseAllocationSize;
            const size_t endBufferIndex = std::min(((offset + size) / baseAllocationSize) + 1, buffers.size());
 
            for(size_t bufferIndex = startBufferIndex; bufferIndex < endBufferIndex; bufferIndex++) {
                BufferType& buffer = buffers[bufferIndex];
                buffer.invalidate();
            }
        }
 
    private:
        void allocateBuffers(const size_t length) {
            const size_t numBuffers = (length + baseAllocationSize - 1) / baseAllocationSize;
            const size_t currentNumBuffers = buffers.size();
            if(currentNumBuffers >= numBuffers) {
                return;
            }
 
            const size_t numBuffersToAllocate = numBuffers - currentNumBuffers;
            for(size_t i = 0; i < numBuffersToAllocate; i++) {
                buffers.emplace_back(*device, baseAllocationSize, "");
            }
            mLength = baseAllocationSize * buffers.size() / alignment;
        }
    };
 
    template<typename T> using TGPUUniformBufferPool = TGPUBufferPool<T, GPUDynamicUniformBuffer>;
 
    template<typename T> using TGPUStorageBufferPool = TGPUBufferPool<T, GPUDynamicStorageBuffer>;
 
    template<typename T> using TGPUIndirectBufferPool = TGPUBufferPool<T, GPUDynamicIndirectBuffer>;
 
    template<typename T> using TGPUBindlessBufferPool = TGPUBufferPool<T, GPUDynamicBindlessBuffer>;
 
    using GPUIndirectBufferPool = TGPUIndirectBufferPool<GPUDrawIndirectCommand>;
 
} // namespace CeresEngine