BinaryFile.h

Go to the documentation of this file.

/*--------------------------------------------------------------------------------------------
Copyright (c) 2019, NDEVR LLC
tyler.parke@ndevr.org
  __    __   ____     _____ __     __  _______
 |  \  |  | |  __ \  |  ___|\ \   / / |   __  \
 |   \ |  | | |  \ \ | |___  \ \ / /  |  |__)  |
 |  . \|  | | |__/ / | |___   \ V /   |   _   /
 |  |\    |_|_____/__|_____|___\_/____|  | \  \
 |__| \__________________________________|  \__\
 
Subject to the terms of the Enterprise+ Agreement, NDEVR hereby grants
Licensee a limited, non-exclusive, non-transferable, royalty-free license
(without the right to sublicense) to use the API solely for the purpose of
Licensee's internal development efforts to develop applications for which
the API was provided.
 
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
 
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
 
Library: Base
File: BinaryFile
Included in API: True
Author(s): Tyler Parke
 *-----------------------------------------------------------------------------------------**/
#pragma once
#include "DLLInfo.h"
#include <NDEVR/File.h>
#include <NDEVR/String.h>
#include <NDEVR/Dictionary.h>
#include <NDEVR/Bounds.h>
#include <NDEVR/Compressor.h>
#include <fstream>
namespace NDEVR
{
    /**----------------------------------------------------------------------------
    Converts an object from big endian to little endian. Does not consider structue
        of object so will need to be called for each sub-part of a non-primitive type
    ----------------------------------------------------------------------------*/
    template <class t_type>
    t_type ChangeEndian(t_type in)
    {
        char* const p = reinterpret_cast<char*>(&in);
        for (size_t i = 0; i < sizeof(t_type) / 2; ++i)
            std::swap(p[i], p[sizeof(t_type) - i - 1]);
        return in;
    }
    /**----------------------------------------------------------------------------
    \brief Logic for reading or writing to a binary file including logic for 
        compressing or decompressing the file
    ----------------------------------------------------------------------------*/
    class NDEVR_BASE_API BinaryFile
    {
    public:
 
        explicit BinaryFile(const File& file)
            : m_file(file)
        {}
        ~BinaryFile()
        {
            close();
        }
        void open(bool read, bool safe)
        {
            m_is_read = read;
            if (read)
                m_file.open(safe ? File::e_binary_read_safe : File::e_binary_read);
            else
                m_file.open(safe ? File::e_binary_write_safe : File::e_binary_write);
        }
        void cachedOpen(bool read)
        {
            m_is_read = read;
            m_using_cache = true;
            m_cache_location = 0;
            if (read)
            {
                m_file.open(File::e_binary_read);
                m_file.invalidateCache();
                uint08 size = m_file.fileSize();
                m_cached_data.setSize(size);
                fread(m_cached_data.begin(), 1, m_cached_data.size(), m_file.filePtr());
                m_file.close();
            }
            else
            {
                m_cached_data.clear();
            }
 
        }
        void close()
        {
            if (m_is_read)
            {
                if (!m_using_cache)
                    m_file.close();
                else if (m_multitreaded_compression)
                    uncompressAll();
            }
            else
            {
                if (m_using_cache)
                {
                    m_file.open(File::e_binary_write);
                    fwrite(m_cached_data.begin(), 1, m_cached_data.size(), m_file.filePtr());
                    m_file.close();
                }
            }
 
        }
        const Buffer<uint01, uint08>& cachedData() const
        {
            return m_cached_data;
        }
        //Data writing
        template<class t_type>
        void write(const t_type& data)
        {
            if (m_using_cache)
            {
 
                if (m_cache_location == m_cached_data.size())
                    m_cached_data.addAll((uint01*)&data, sizeof(data));
                else
                    m_cached_data.setAll((uint01*)&data, m_cache_location, sizeof(data));
                m_cache_location += sizeof(data);
            }
            else
            {
                fwrite(&data, sizeof(t_type), 1, m_file.filePtr());
            }
 
        }
 
        template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
        typename std::enable_if<!ObjectInfo<t_type>::Buffer>::type write(const Buffer<t_type, t_index_type, t_memory_allocator, t_memory_manager>& data, CompressionMode compression_mode)
        {
            if (data.size() != 0)
            {
                BinaryCompressionObject binary = createCompressionObject();
                binary.buffer_size = data.size();
                binary.compression_mode = Compressor::PickCompressionMode<t_type>(compression_mode);
                binary.uncompressed_data = (uint01*)data.ptr();
                binary.uncompressed_size = data.memSize();
                if (binary.compression_mode == e_string_compression)
                {
                    m_compressed_data.setSize(sizeof(Bounds<1, uint04>));
                    binary.compressed_size = sizeof(Bounds<1, uint04>);
                }
                else if (binary.compression_mode == e_string_reference)
                {
                    m_compressed_data.clear();
                    binary.compressed_size = 0;
                }
                else
                {
                    m_compressed_data.setSize(data.memSize());
                    binary.compressed_size = data.memSize();
                }
                binary.compressed_data = m_compressed_data.begin();
                Compressor::CompressData(binary);
                writeCompression(binary);
            }
            else
            {
                //lib_assert(m_large_compression || cast<uint08>(data.size()) < cast<uint08>(cast<uint04>::Max), "Bad Data write");
                if (m_large_compression)
                    write(cast<uint08>(data.size()));
                else
                    write(cast<uint04>(data.size()));
            }
        }
        template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
        typename std::enable_if<ObjectInfo<t_type>::Buffer>::type write(const Buffer<t_type, t_index_type, t_memory_allocator, t_memory_manager>& data, CompressionMode compression_mode)
        {
            uint04 total_size = 0;
            Buffer<uint04> sizes;
            sizes.setSize(data.size());
            for (uint04 i = 0; i < data.size(); i++)
            {
                uint04 size = data[i].size();;
                sizes[i] = size;
                total_size += size;
            }
            Buffer<decltype(t_type::Type())> combined_data(total_size);
            for (uint04 i = 0; i < data.size(); i++)
            {
                combined_data.addAll(data[i]);
            }
            write(sizes, compression_mode);
            write(combined_data, compression_mode);
        }
        void writeRawData(const String& data, uint04 size, char fill_space = '\0')
        {
            if (m_using_cache)
            {
 
                if (m_cache_location == m_cached_data.size())
                    m_cached_data.addAll((uint01*)data.begin(), getMin(data.size(), size));
                else
                    m_cached_data.setAll((uint01*)data.begin(), m_cache_location, getMin(data.size(), size));
                m_cache_location += getMin(data.size(), size);
                if (size > data.size())
                {
                    uint04 add_size = size - data.size();
                    if (m_cache_location == m_cached_data.size())
                        m_cached_data.setAllToValue(fill_space, m_cache_location, add_size);
                    else
                        m_cached_data.addAndFillSpace(add_size, fill_space);
                    m_cache_location += add_size;
                }
 
            }
            else
            {
                fwrite(data.begin(), data.size(), 1, m_file.filePtr());
                if (size > data.size())
                {
                    uint04 add_size = size - data.size();
                    for(uint04 i = 0; i < add_size; i++)
                        fwrite(&fill_space, add_size, 1, m_file.filePtr());
                }
            }
        }
        //data reading
        template<class t_type>
        t_type read()
        {
            t_type data;
            if (m_using_cache)
            {
                data = *((t_type*)(&m_cached_data[m_cache_location]));
                m_cache_location += cast<uint08>(sizeof(t_type));
            }
            else
            {
                fread(&data, sizeof(t_type), 1, m_file.filePtr());
            }
            return data;
        }
        void readString(String& string, char terminator = '\0')
        {
            if (m_using_cache)
            {
                for (;;)
                {
                    char c = *((char*)(&m_cached_data[m_cache_location++]));
                    if (c == terminator || m_cache_location >= m_cached_data.size())
                        return;
                    string.add(c);
                }
            }
            else
            {
                for (;;)
                {
                    char c = cast<char>(fgetc(m_file.filePtr()));
                    if (c == terminator || feof(m_file.filePtr()))
                        return;
                    string.add(c);
                }
            }
        }
        uint04 readString(char* string, char terminator = '\0')
        {
            uint04 size = 0;
            if (m_using_cache)
            {
                for (;;)
                {
                    char c = *((char*)(&m_cached_data[m_cache_location++]));
                    if (c == terminator || m_cache_location >= m_cached_data.size())
                        break;
                    string[size++] = c;
                }
            }
            else
            {
                for (;;)
                {
                    char c = cast<char>(fgetc(m_file.filePtr()));
                    if (c == terminator || feof(m_file.filePtr()))
                        break;
                    string[size++] = c;
                }
            }
            string[size] = '\0';
            return size;
        }
        String readData(uint04 max_size)
        {
            String string;
            string.setSize(max_size);
            uint04 actual_size = readData(string.begin(), max_size);
            string.setSize(actual_size);
            return string;
        }
        uint04 readData(char* string, uint04 max_size)
        {
            if (m_using_cache)
            {
                for (uint04 i = 0; i < max_size; i++)
                {
                    char c = *((char*)(&m_cached_data[m_cache_location++]));
                    if (m_cache_location >= m_cached_data.size())
                        return i;
                    string[i] = c;
                }
            }
            else
            {
                for (uint04 i = 0; i < max_size; i++)
                {
                    char c = cast<char>(fgetc(m_file.filePtr()));
                    if (feof(m_file.filePtr()))
                        return i;
                    string[i] = c;
                }
            }
            return max_size;
        }
        template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
        void read(Buffer<t_type, t_index_type, t_memory_allocator, t_memory_manager>& data)
        {
            bool allow_multithread = m_multitreaded_compression;
            read(data, allow_multithread);
        }
        template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
        void readNow(Buffer<t_type, t_index_type, t_memory_allocator, t_memory_manager>& data)
        {
            bool allow_multithread = false;
            read(data, allow_multithread);
        }
 
        template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
        typename std::enable_if<ObjectInfo<t_type>::Buffer>::type read(Buffer<t_type, t_index_type, t_memory_allocator, t_memory_manager>& data, bool& allow_multithread)
        {
            Buffer<uint04> sizes;
            Buffer<decltype(t_type::Type())> combined_data;
            //if (ObjectInfo<t_type>::String)
            allow_multithread = false;
            read(sizes, allow_multithread);
            read(combined_data, allow_multithread);
            data.setSize(sizes.size());
            uint04 current_offset = 0;
            for (uint04 i = 0; i < sizes.size(); i++)
            {
                data[i].clear();
                if (sizes[i] > 0)
                    data[i].addAll(&combined_data[0] + current_offset, sizes[i]);
                current_offset += sizes[i];
            }
        }
 
        template<class t_index_type, class t_memory_allocator, class t_memory_manager>
        void readStringBuffer(Buffer<String, t_index_type, t_memory_allocator, t_memory_manager>& data, bool& allow_multithread)
        {
            Buffer<uint04> indices;
            //if (ObjectInfo<t_type>::String)
            allow_multithread = false;
            read(indices, allow_multithread);
            data.setSize(indices.size());
            for (uint04 i = 0; i < indices.size(); i++)
            {
                Bounds<1, uint04> bounds = m_string_size_info[indices[i]];
                if (bounds.span() > 0U)
                {
                    data[i] = m_strings.substr(bounds[MIN], bounds[MAX]);
                }
                else
                {
                    data[i].clear();
                }
            }
        }
 
        template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
        typename std::enable_if<!ObjectInfo<t_type>::Buffer>::type read(Buffer<t_type, t_index_type, t_memory_allocator, t_memory_manager>& data, bool& allow_multithread)
        {
            BinaryCompressionObject binary = createCompressionObject();
            if (m_large_compression)
                binary.uncompressed_size = read<uint08>();
            else
                binary.uncompressed_size = read<uint04>();
            //lib_assert(binary.uncompressed_size < Constant<sint04>::Max, "Large number read");
            data.setSize(cast<t_index_type>(binary.uncompressed_size));
            binary.uncompressed_size = data.memSize();
            if (data.size() != 0)
            {
                binary.uncompressed_data = (uint01*)data.ptr();
                if (m_large_compression)
                    binary.compressed_size = read<uint08>();
                else
                    binary.compressed_size = read<uint04>();
                if (m_using_cache)
                {
                    binary.compression_mode = cast<CompressionMode>(m_cached_data[m_cache_location]);
                    binary.compressed_size -= 1;
                    if (binary.compression_mode != CompressionMode::e_string_reference)
                    {
                        binary.compressed_data = &m_cached_data[m_cache_location + 1];
 
                        allow_multithread &= binary.compression_mode != CompressionMode::e_string_compression;
                        allow_multithread &= binary.compression_mode != CompressionMode::e_no_compression;
                        if (m_multitreaded_compression && allow_multithread)
                            m_compressions.add(binary);
                        else
                            Compressor::DecompressData(binary);
                        m_cache_location += binary.compressed_size + 1;
                    }
                    else
                    {
                        Compressor::DecompressData(binary);
                        m_cache_location++;
                    }
                }
                else
                {
                    if (binary.compression_mode != CompressionMode::e_string_reference)
                    {
                        m_compressed_data.setSize(binary.compressed_size);
#ifdef USE_ASSERTIONS
                        size_t fsize = fread(&m_compressed_data[0], 1, m_compressed_data.size(), m_file.filePtr());
                        lib_assert(m_compressed_data.size() == cast<uint08>(fsize), "Bad read size");
#else
                        fread(&m_compressed_data[0], 1, m_compressed_data.size(), m_file.filePtr());
#endif
                        binary.compression_mode = cast<CompressionMode>(m_compressed_data[0]);
                        binary.compressed_data = &m_compressed_data[1];
                        binary.compressed_size -= 1;
                        Compressor::DecompressData(binary);
                        m_cache_location += binary.compressed_size + 1;
                    }
                    else
                    {
                        Compressor::DecompressData(binary);
                        m_cache_location++;
                    }
                }
            }
        }
#ifdef _WIN32
#define ndevr_ftell _ftelli64
#define ndevr_fseek _fseeki64
#else
#define ndevr_ftell ftell
#define ndevr_fseek fseek
#endif
        uint08 position()
        {
            if (m_using_cache)
                return m_cache_location;
            else
                return cast<uint08>(ndevr_ftell(m_file.filePtr()));
        }
        fltp04 percent()
        {
            if (m_using_cache)
            {
                return cast<fltp04>(m_cache_location) / cast<fltp04>(m_cached_data.size());
            }
            else
            {
                if (IsInvalid(m_file_size))
                    m_file_size = m_file.fileSize();
                return cast<fltp04>(cast<uint08>(ndevr_ftell(m_file.filePtr()))) / cast<fltp04>(m_file_size);
            }
        }
        void seek(uint08 location)
        {
            if (m_using_cache)
                m_cache_location = location;
            else
                ndevr_fseek(m_file.filePtr(), cast<long long>(location), SEEK_SET);
        }
        BinaryCompressionObject createCompressionObject();
        void setUseLargeCompression(bool use_large_compression) { m_large_compression = use_large_compression; }
        bool isFinished();
        void writeCompression(BinaryCompressionObject& compression_object);
        void writeStringData(CompressionMode compression_mode);
        void readStringData(bool has_reference_table);
        void uncompressAll();
        void uncompressSections(uint04 start, uint04 count);
        uint08 fileSize() const
        {
            if (m_using_cache)
                return m_cached_data.size();
            if (IsInvalid(m_file_size))
                m_file_size = m_file.fileSize();
            return m_file_size;
        }
    protected:
        File m_file;
        String m_strings;
        Dictionary<String, uint04> m_string_reference;
        Buffer<Bounds<1, uint04>> m_string_size_info;
        fltp08 fltp_error = Constant<fltp08>::Invalid;
        Buffer<uint01, uint08> m_compressed_data;
        Buffer<uint01, uint08> m_cached_data;
        Buffer<BinaryCompressionObject> m_compressions;
        uint08 m_cache_location = Constant<uint08>::Invalid;
        mutable uint08 m_file_size = Constant<uint08>::Invalid;
        bool m_using_cache = false;
        bool m_is_read = false;
        bool m_large_compression = false;
        bool m_multitreaded_compression = true;
    };
    template class NDEVR_BASE_API StringStream<CompressionMode>;
}