BinaryFile.h

/*--------------------------------------------------------------------------------------------
Copyright (c) 2019, NDEVR LLC
tyler.parke@ndevr.org
  __    __   ____     _____ __     __  _______
 |  \  |  | |  __ \  |  ___|\ \   / / |   __  \
 |   \ |  | | |  \ \ | |___  \ \ / /  |  |__)  |
 |  . \|  | | |__/ / | |___   \ V /   |   _   /
 |  |\    |_|_____/__|_____|___\_/____|  | \  \
 |__| \__________________________________|  \__\
 
Subject to the terms of the Enterprise+ Agreement, NDEVR hereby grants
Licensee a limited, non-exclusive, non-transferable, royalty-free license
(without the right to sublicense) to use the API solely for the purpose of
Licensee's internal development efforts to develop applications for which
the API was provided.
 
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
 
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
 
Library: Base
File: BinaryFile
Included in API: True
Author(s): Tyler Parke
 *-----------------------------------------------------------------------------------------**/
#pragma once
#include "DLLInfo.h"
#include <NDEVR/File.h>
#include <NDEVR/String.h>
#include <NDEVR/Dictionary.h>
#include <NDEVR/Bounds.h>
#include <NDEVR/Compressor.h>
#include <fstream>
#include <cstddef>
#include <type_traits>
namespace NDEVR
{
#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && defined(__ORDER_BIG_ENDIAN__)
#if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
    static constexpr bool kNativeLittleEndian = true; 
#elif (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
    static constexpr bool kNativeLittleEndian = false; 
#else
#error Unsupported native endianness
#endif
 
#elif defined(_WIN32) || defined(_WIN64)
    static constexpr bool kNativeLittleEndian = true; 
 
#elif defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__)
    static constexpr bool kNativeLittleEndian = true; 
 
#elif defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM) || defined(_M_ARM64)
    static constexpr bool kNativeLittleEndian = true; 
 
#elif defined(__ARMEB__) || defined(__AARCH64EB__)
    static constexpr bool kNativeLittleEndian = false; 
 
#else
#error Cannot determine endianness at compile time on this platform (C++17)
#endif

    template <class T>
    constexpr T ByteSwap(T v) noexcept
    {
        static_assert(std::is_integral<T>::value, "T must be an integral type");
        static_assert(!std::is_same<T, bool>::value, "bool is not supported");
 
        using U = typename std::make_unsigned<T>::type;
        U x = static_cast<U>(v);
        U y = 0;
 
        for (std::size_t i = 0; i < sizeof(T); ++i)
        {
            y = (y << 8) | (x & static_cast<U>(0xFF));
            x >>= 8;
        }
 
        return static_cast<T>(y);
    }

    template <class T>
    constexpr T _LE(T v) noexcept
    {
        static_assert(std::is_integral<T>::value, "T must be an integral type");
        static_assert(!std::is_same<T, bool>::value, "bool is not supported");
 
        return kNativeLittleEndian ? v : ByteSwap(v);
    }

    template <class T>
    constexpr T _BE(T v) noexcept
    {
        static_assert(std::is_integral<T>::value, "T must be an integral type");
        static_assert(!std::is_same<T, bool>::value, "bool is not supported");
 
        return kNativeLittleEndian ? ByteSwap(v) : v;
    }

    template <class t_type>
    constexpr t_type ChangeEndian(t_type in)
    {
        return ByteSwap(in);
    }

    class NDEVR_BASE_API BinaryFile
    {
    public:

        explicit BinaryFile(const File& file)
            : m_file(file)
        {}

        explicit BinaryFile()
            : m_file()
            , m_cache_location(0U)
            , m_using_cache(true)
            , m_is_read(false)
        {}

        explicit BinaryFile(const HighCapacityBuffer<uint01>& data)
            : m_cached_data(data)
            , m_cache_location(0U)
            , m_using_cache(true)
            , m_is_read(false)
        {}

        explicit BinaryFile(HighCapacityBuffer<uint01>&& data)
            : m_cached_data(std::move(data))
            , m_cache_location(0U)
            , m_using_cache(true)
            , m_is_read(false)
        {
        }

        explicit BinaryFile(const uint01* data, uint08 size)
            : m_cache_location(0U)
            , m_using_cache(true)
            , m_is_read(false)
        {
            m_cached_data.addAll(data, size);
        }

        explicit BinaryFile(const Buffer<uint01>& data)
            : m_cache_location(0U)
            , m_using_cache(true)
            , m_is_read(false)
        {
            m_cached_data.addAll(data);
        }

        BinaryFile(BinaryFile&& bf) noexcept;
        BinaryFile& operator=(BinaryFile&& bf) noexcept;
        ~BinaryFile()
        {
            close();
        }

        void open(bool read, bool safe);
        void cachedOpen(bool read);
        void close();
        const HighCapacityBuffer<uint01>& cachedData() const
        {
            return m_cached_data;
        }

        HighCapacityBuffer<uint01>& cachedData()
        {
            return m_cached_data;
        }

        //Data writing
        template<class t_type>
        typename std::enable_if<!ObjectInfo<t_type>::Buffer>::type write(const t_type& data)
        {
            writeDirect((uint01*)&data, sizeof(t_type));
        }

        template<class t_type, class t_memory_manager>
        typename std::enable_if<!ObjectInfo<t_type>::Buffer>::type write(const Buffer<t_type, t_memory_manager>& data, CompressionMode compression_mode = CompressionMode::e_no_compression)
        {
            if (data.size() != 0)
            {
                BinaryCompressionObject binary = createCompressionObject();
                binary.buffer_size = data.size();
                binary.object_type = GetTypeInfo<t_type>();
                binary.compression_mode = Compressor::PickCompressionMode<t_type>(compression_mode);
                binary.uncompressed_data = (uint01*)data.ptr();
                binary.uncompressed_size = data.memSize();
                if (binary.compression_mode == e_string_compression)
                {
                    m_compressed_data.setSize(sizeof(Bounds<1, uint04>));
                    binary.compressed_size = sizeof(Bounds<1, uint04>);
                }
                else if (binary.compression_mode == e_string_reference)
                {
                    m_compressed_data.clear();
                    binary.compressed_size = 0;
                }
                else
                {
                    m_compressed_data.setSize(data.memSize());
                    binary.compressed_size = data.memSize();
                }
                binary.compressed_data = m_compressed_data.begin();
                Compressor::CompressData(binary);
                writeCompression(binary);
            }
            else
            {
                //lib_assert(m_large_compression || cast<uint08>(data.size()) < cast<uint08>(cast<uint04>::Max), "Bad Data write");
                if (m_large_compression)
                    write(cast<uint08>(data.size()));
                else
                    write(cast<uint04>(data.size()));
            }
        }

        template<class t_type, class t_memory_manager>
        typename std::enable_if<ObjectInfo<t_type>::Buffer>::type write(const Buffer<t_type, t_memory_manager>& data, CompressionMode compression_mode)
        {
            uint04 total_size = 0;
            Buffer<uint04> sizes;
            sizes.setSize(data.size());
            for (uint04 i = 0; i < data.size(); i++)
            {
                uint04 size = data[i].size();;
                sizes[i] = size;
                total_size += size;
            }
            Buffer<decltype(t_type::Type())> combined_data(total_size);
            for (uint04 i = 0; i < data.size(); i++)
            {
                combined_data.addAll(data[i]);
            }
            write(sizes, compression_mode);
            write(combined_data, compression_mode);
        }

        void write(const StringView& data, CompressionMode compression_mode)
        {
            if (data.size() != 0)
            {
                BinaryCompressionObject binary = createCompressionObject();
                binary.buffer_size = data.size();
                binary.object_type = GetTypeInfo<String>();
                binary.compression_mode = Compressor::PickCompressionMode<String>(compression_mode);
                binary.uncompressed_data = (uint01*)data.begin();
                binary.uncompressed_size = data.size();
                if (binary.compression_mode == e_string_compression)
                {
                    m_compressed_data.setSize(sizeof(Bounds<1, uint04>));
                    binary.compressed_size = sizeof(Bounds<1, uint04>);
                }
                else if (binary.compression_mode == e_string_reference)
                {
                    m_compressed_data.clear();
                    binary.compressed_size = 0;
                }
                else
                {
                    m_compressed_data.setSize(data.size());
                    binary.compressed_size = data.size();
                }
                binary.compressed_data = m_compressed_data.begin();
                Compressor::CompressData(binary);
                writeCompression(binary);
            }
            else
            {
                //lib_assert(m_large_compression || cast<uint08>(data.size()) < cast<uint08>(cast<uint04>::Max), "Bad Data write");
                if (m_large_compression)
                    write(cast<uint08>(data.size()));
                else
                    write(cast<uint04>(data.size()));
            }
        }

        void writeDirect(const uint01* ptr, uint08 size);
        uint08 readDirect(uint01* ptr, uint08 size);
        void writeRawData(const StringView& data, uint04 size, char fill_space = '\0')
        {
            if (m_using_cache)
            {
 
                if (m_cache_location == m_cached_data.size())
                    m_cached_data.addAll((uint01*)data.begin(), getMin(data.size(), size));
                else
                    m_cached_data.setAll((uint01*)data.begin(), m_cache_location, getMin(data.size(), size));
                m_cache_location += getMin(data.size(), size);
                if (size > data.size())
                {
                    uint04 add_size = size - data.size();
                    if (m_cache_location == m_cached_data.size())
                        m_cached_data.setAllToValue(fill_space, m_cache_location, add_size);
                    else
                        m_cached_data.addAndFillSpace(add_size, fill_space);
                    m_cache_location += add_size;
                }
 
            }
            else
            {
                fwrite(data.begin(), data.size(), 1, m_file.filePtr());
                if (size > data.size())
                {
                    uint04 add_size = size - data.size();
                    for(uint04 i = 0; i < add_size; i++)
                        fwrite(&fill_space, add_size, 1, m_file.filePtr());
                }
            }
        }

        template<class t_type>
        [[nodiscard]] bool canRead() const
        {
            if (m_using_cache)
                return m_cache_location + sizeof(t_type) <= m_cached_data.size();
            else
                return position() + sizeof(t_type) <= m_file.size();
 
        }

        //data reading
        template<class t_type>
        [[nodiscard]] t_type read()
        {
            t_type data;
            if (m_using_cache)
            {
                data = *((t_type*)(&m_cached_data[m_cache_location]));
                m_cache_location += cast<uint08>(sizeof(t_type));
            }
            else
            {
                fread(&data, sizeof(t_type), 1, m_file.filePtr());
            }
            return data;
        }

        template<class t_type>
        void skip()
        {
            if (m_using_cache)
                m_cache_location += cast<uint08>(sizeof(t_type));
            else
                fseek(m_file.filePtr(), sizeof(t_type), SEEK_CUR);
        }

        void writeTerminatingString(const StringView& string, char terminator = '\0');
        void readString(String& string, char terminator = '\0');
        void switchToFile(File file, bool is_write, bool is_safe);
        uint04 readString(char* string, char terminator = '\0');
        String readData(uint04 max_size);
        uint04 readData(char* string, uint04 max_size);
        template<class t_type, class t_memory_manager>
        void read(Buffer<t_type, t_memory_manager>& data)
        {
            bool allow_multithread = m_multithreaded_compression;
            read(data, allow_multithread);
        }

        template<class t_type, class t_memory_manager>
        void readNow(Buffer<t_type, t_memory_manager>& data)
        {
            bool allow_multithread = false;
            read(data, allow_multithread);
        }

        template<class t_type, class t_memory_manager>
        typename std::enable_if<ObjectInfo<t_type>::Buffer>::type read(Buffer<t_type, t_memory_manager>& data, bool& allow_multithread)
        {
            Buffer<uint04> sizes;
            Buffer<decltype(t_type::Type())> combined_data;
            //if (ObjectInfo<t_type>::String)
            allow_multithread = false;
            read(sizes, allow_multithread);
            read(combined_data, allow_multithread);
            data.setSize(sizes.size());
            uint04 current_offset = 0;
            for (uint04 i = 0; i < sizes.size(); i++)
            {
                data[i].clear();
                if (sizes[i] > 0)
                    data[i].addAll(&combined_data[0] + current_offset, sizes[i]);
                current_offset += sizes[i];
            }
        }

        template<class t_memory_manager>
        void readStringBuffer(Buffer<String, t_memory_manager>& data, bool& allow_multithread)
        {
            Buffer<uint04> indices;
            //if (ObjectInfo<t_type>::String)
            allow_multithread = false;
            read(indices, allow_multithread);
            data.setSize(indices.size());
            for (uint04 i = 0; i < indices.size(); i++)
            {
                Bounds<1, uint04> bounds = m_string_size_info[indices[i]];
                if (bounds.span() > 0U)
                    data[i] = m_strings.substr(bounds[MIN], bounds[MAX]);
                else
                    data[i].clear();
            }
        }

        template<class t_type, class t_memory_manager>
        typename std::enable_if<!ObjectInfo<t_type>::Buffer>::type read(Buffer<t_type, t_memory_manager>& data, bool& allow_multithread)
        {
            BinaryCompressionObject binary = createCompressionObject();
            if (m_large_compression)
                binary.uncompressed_size = read<uint08>();
            else
                binary.uncompressed_size = read<uint04>();
            using t_index_type = typename t_memory_manager::index_type;
            //lib_assert(binary.uncompressed_size < Constant<sint04>::Max, "Large number read");
            data.setSize(cast<t_index_type>(binary.uncompressed_size));
            binary.uncompressed_size = data.memSize();
            if (data.size() != 0)
            {
                binary.uncompressed_data = (uint01*)data.ptr();
                if (m_large_compression)
                    binary.compressed_size = read<uint08>();
                else
                    binary.compressed_size = read<uint04>();
                if (m_using_cache)
                {
                    binary.compression_mode = cast<CompressionMode>(m_cached_data[m_cache_location]);
                    binary.compressed_size -= 1;
                    if (binary.compression_mode != CompressionMode::e_string_reference)
                    {
                        binary.compressed_data = &m_cached_data[m_cache_location + 1];
 
                        allow_multithread &= binary.compression_mode != CompressionMode::e_string_compression;
                        allow_multithread &= binary.compression_mode != CompressionMode::e_no_compression;
                        if (m_multithreaded_compression && allow_multithread)
                            m_compressions.add(binary);
                        else
                            Compressor::DecompressData(binary);
                        m_cache_location += binary.compressed_size + 1;
                    }
                    else
                    {
                        Compressor::DecompressData(binary);
                        m_cache_location++;
                    }
                }
                else
                {
                    if (binary.compression_mode != CompressionMode::e_string_reference)
                    {
                        m_compressed_data.setSize(binary.compressed_size);
#ifdef USE_ASSERTIONS
                        size_t fsize = fread(&m_compressed_data[0], 1, m_compressed_data.size(), m_file.filePtr());
                        lib_assert(m_compressed_data.size() == cast<uint08>(fsize), "Bad read size");
#else
                        fread(&m_compressed_data[0], 1, m_compressed_data.size(), m_file.filePtr());
#endif
                        binary.compression_mode = cast<CompressionMode>(m_compressed_data[0]);
                        binary.compressed_data = &m_compressed_data[1];
                        binary.compressed_size -= 1;
                        Compressor::DecompressData(binary);
                        m_cache_location += binary.compressed_size + 1;
                    }
                    else
                    {
                        Compressor::DecompressData(binary);
                        m_cache_location++;
                    }
                }
            }
        }

        uint08 position() const;
        fltp04 percent() const;
        void seek(uint08 location);
        BinaryCompressionObject createCompressionObject();
        void setUseLargeCompression(bool use_large_compression) { m_large_compression = use_large_compression; }
        bool isFinished();
        void writeCompression(BinaryCompressionObject& compression_object);
        void writeStringData(CompressionMode compression_mode);
        void readStringData(bool has_reference_table);
        void uncompressAll();
        void uncompressSections(uint04 start, uint04 count);
        bool readDirectlyTo(File file);
        bool writeDirectlyFrom(File file);
        const File& file() const { return m_file; }
        File& file() { return m_file; }
        uint08 fileSize() const;
        void setMaxBufferSize(uint08 max_buffer_size)
        {
            m_max_buffer_size = max_buffer_size;
        }
    protected:
        File m_file; 
        String m_strings; 
        uint08 m_max_buffer_size = Constant<uint04>::Max; 
        Dictionary<String, uint04> m_string_reference; 
        Buffer<Bounds<1, uint04>> m_string_size_info; 
        fltp08 fltp_error = Constant<fltp08>::Invalid; 
        HighCapacityBuffer<uint01> m_compressed_data; 
        HighCapacityBuffer<uint01> m_cached_data; 
        Buffer<BinaryCompressionObject> m_compressions; 
        uint08 m_cache_location = Constant<uint08>::Invalid; 
        mutable uint08 m_file_size = Constant<uint08>::Invalid; 
        bool m_using_cache = false; 
        bool m_is_read = false; 
        bool m_large_compression = false; 
        bool m_multithreaded_compression = true; 
 
    };
    template class NDEVR_BASE_API StringStream<CompressionMode>;
}