34#include <NDEVR/File.h>
35#include <NDEVR/String.h>
36#include <NDEVR/Dictionary.h>
37#include <NDEVR/Bounds.h>
38#include <NDEVR/Compressor.h>
42 template <
class t_type>
45 char*
const p =
reinterpret_cast<char*
>(&in);
46 for (
size_t i = 0; i <
sizeof(t_type) / 2; ++i)
47 std::swap(p[i], p[
sizeof(t_type) - i - 1]);
61 void open(
bool read,
bool safe)
65 m_file.open(safe ? File::e_binary_read_safe : File::e_binary_read);
67 m_file.open(safe ? File::e_binary_write_safe : File::e_binary_write);
76 m_file.open(File::e_binary_read);
77 m_file.invalidateCache();
78 uint08 size = m_file.fileSize();
79 m_cached_data.setSize(size);
80 fread(m_cached_data.begin(), 1, m_cached_data.size(), m_file.filePtr());
85 m_cached_data.clear();
95 else if (m_multitreaded_compression)
102 m_file.open(File::e_binary_write);
103 fwrite(m_cached_data.begin(), 1, m_cached_data.size(), m_file.filePtr());
111 return m_cached_data;
114 template<
class t_type>
120 if (m_cache_location == m_cached_data.size())
121 m_cached_data.addAll((
uint01*)&data,
sizeof(data));
123 m_cached_data.setAll((
uint01*)&data, m_cache_location,
sizeof(data));
124 m_cache_location +=
sizeof(data);
128 fwrite(&data,
sizeof(t_type), 1, m_file.filePtr());
133 template<
class t_type,
class t_index_type,
class t_memory_allocator,
class t_memory_manager>
136 if (data.
size() != 0)
140 binary.
compression_mode = Compressor::PickCompressionMode<t_type>(compression_mode);
150 m_compressed_data.clear();
155 m_compressed_data.setSize(data.
memSize());
159 Compressor::CompressData(binary);
160 writeCompression(binary);
165 if (m_large_compression)
166 write(cast<uint08>(data.
size()));
168 write(cast<uint04>(data.
size()));
171 template<
class t_type,
class t_index_type,
class t_memory_allocator,
class t_memory_manager>
183 Buffer<
decltype(t_type::Type())> combined_data(total_size);
186 combined_data.addAll(data[i]);
188 write(sizes, compression_mode);
189 write(combined_data, compression_mode);
196 if (m_cache_location == m_cached_data.size())
200 m_cache_location +=
getMin(data.
size(), size);
201 if (size > data.
size())
204 if (m_cache_location == m_cached_data.size())
205 m_cached_data.setAllToValue(fill_space, m_cache_location, add_size);
207 m_cached_data.addAndFillSpace(add_size, fill_space);
208 m_cache_location += add_size;
214 fwrite(data.
begin(), data.
size(), 1, m_file.filePtr());
215 if (size > data.
size())
218 for(
uint04 i = 0; i < add_size; i++)
219 fwrite(&fill_space, add_size, 1, m_file.filePtr());
224 template<
class t_type>
230 data = *((t_type*)(&m_cached_data[m_cache_location]));
231 m_cache_location += cast<uint08>(
sizeof(t_type));
235 fread(&data,
sizeof(t_type), 1, m_file.filePtr());
245 char c = *((
char*)(&m_cached_data[m_cache_location++]));
246 if (c == terminator || m_cache_location >= m_cached_data.size())
255 char c = cast<char>(fgetc(m_file.filePtr()));
256 if (c == terminator || feof(m_file.filePtr()))
269 char c = *((
char*)(&m_cached_data[m_cache_location++]));
270 if (c == terminator || m_cache_location >= m_cached_data.size())
279 char c = cast<char>(fgetc(m_file.filePtr()));
280 if (c == terminator || feof(m_file.filePtr()))
292 uint04 actual_size = readData(
string.begin(), max_size);
293 string.setSize(actual_size);
300 for (
uint04 i = 0; i < max_size; i++)
302 char c = *((
char*)(&m_cached_data[m_cache_location++]));
303 if (m_cache_location >= m_cached_data.size())
310 for (
uint04 i = 0; i < max_size; i++)
312 char c = cast<char>(fgetc(m_file.filePtr()));
313 if (feof(m_file.filePtr()))
320 template<
class t_type,
class t_index_type,
class t_memory_allocator,
class t_memory_manager>
323 bool allow_multithread = m_multitreaded_compression;
324 read(data, allow_multithread);
326 template<
class t_type,
class t_index_type,
class t_memory_allocator,
class t_memory_manager>
329 bool allow_multithread =
false;
330 read(data, allow_multithread);
333 template<
class t_type,
class t_index_type,
class t_memory_allocator,
class t_memory_manager>
337 Buffer<
decltype(t_type::Type())> combined_data;
339 allow_multithread =
false;
340 read(sizes, allow_multithread);
341 read(combined_data, allow_multithread);
343 uint04 current_offset = 0;
348 data[i].
addAll(&combined_data[0] + current_offset, sizes[i]);
349 current_offset += sizes[i];
353 template<
class t_index_type,
class t_memory_allocator,
class t_memory_manager>
358 allow_multithread =
false;
359 read(indices, allow_multithread);
364 if (bounds.
span() > 0U)
366 data[i] = m_strings.substr(bounds[
MIN], bounds[
MAX]);
375 template<
class t_type,
class t_index_type,
class t_memory_allocator,
class t_memory_manager>
379 if (m_large_compression)
386 if (data.
size() != 0)
389 if (m_large_compression)
395 binary.
compression_mode = cast<CompressionMode>(m_cached_data[m_cache_location]);
401 allow_multithread &= binary.
compression_mode != CompressionMode::e_string_compression;
402 allow_multithread &= binary.
compression_mode != CompressionMode::e_no_compression;
403 if (m_multitreaded_compression && allow_multithread)
404 m_compressions.add(binary);
406 Compressor::DecompressData(binary);
411 Compressor::DecompressData(binary);
421 size_t fsize = fread(&m_compressed_data[0], 1, m_compressed_data.size(), m_file.filePtr());
422 lib_assert(m_compressed_data.size() == cast<uint08>(fsize),
"Bad read size");
424 fread(&m_compressed_data[0], 1, m_compressed_data.size(), m_file.filePtr());
429 Compressor::DecompressData(binary);
434 Compressor::DecompressData(binary);
441#define ndevr_ftell _ftelli64
442#define ndevr_fseek _fseeki64
444#define ndevr_ftell ftell
445#define ndevr_fseek fseek
450 return m_cache_location;
452 return cast<uint08>(
ndevr_ftell(m_file.filePtr()));
458 return cast<fltp04>(m_cache_location) / cast<fltp04>(m_cached_data.size());
462 if (
isNaN(m_file_size))
463 m_file_size = m_file.fileSize();
464 return cast<fltp04>(cast<uint08>(
ndevr_ftell(m_file.filePtr()))) / cast<fltp04>(m_file_size);
470 m_cache_location = location;
472 ndevr_fseek(m_file.filePtr(), cast<long long>(location), SEEK_SET);
479 void readStringData(
bool has_reference_table);
480 void uncompressAll();
485 return m_cached_data.size();
486 if (
isNaN(m_file_size))
487 m_file_size = m_file.fileSize();
501 bool m_using_cache =
false;
502 bool m_is_read =
false;
503 bool m_large_compression =
false;
504 bool m_multitreaded_compression =
true;
#define ndevr_fseek
Definition BinaryFile.h:445
#define ndevr_ftell
Definition BinaryFile.h:444
#define lib_assert(expression, message)
Asserts some logic in the code. Disabled in non debug mode by default. Can be re-enabled in release u...
Definition LibAssert.h:70
Definition BinaryFile.h:51
uint04 readString(char *string, char terminator='\0')
Definition BinaryFile.h:262
void read(Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data)
Definition BinaryFile.h:321
void setUseLargeCompression(bool use_large_compression)
Definition BinaryFile.h:475
std::enable_if<!ObjectInfo< t_type >::Buffer >::type read(Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data, bool &allow_multithread)
Definition BinaryFile.h:376
Buffer< Bounds< 1, uint04 > > m_string_size_info
Definition BinaryFile.h:494
void write(const t_type &data)
Definition BinaryFile.h:115
fltp04 percent()
Definition BinaryFile.h:454
std::enable_if< ObjectInfo< t_type >::Buffer >::type write(const Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data, CompressionMode compression_mode)
Definition BinaryFile.h:172
void seek(uint08 location)
Definition BinaryFile.h:467
std::enable_if<!ObjectInfo< t_type >::Buffer >::type write(const Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data, CompressionMode compression_mode)
Definition BinaryFile.h:134
void close()
Definition BinaryFile.h:89
void open(bool read, bool safe)
Definition BinaryFile.h:61
Dictionary< String, uint04 > m_string_reference
Definition BinaryFile.h:493
String m_strings
Definition BinaryFile.h:492
void cachedOpen(bool read)
Definition BinaryFile.h:69
std::enable_if< ObjectInfo< t_type >::Buffer >::type read(Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data, bool &allow_multithread)
Definition BinaryFile.h:334
void readNow(Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data)
Definition BinaryFile.h:327
uint08 fileSize() const
Definition BinaryFile.h:482
void writeRawData(const String &data, uint04 size, char fill_space='\0')
Definition BinaryFile.h:191
uint08 position()
Definition BinaryFile.h:447
void readStringBuffer(Buffer< String, t_index_type, t_memory_allocator, t_memory_manager > &data, bool &allow_multithread)
Definition BinaryFile.h:354
File m_file
Definition BinaryFile.h:491
Buffer< BinaryCompressionObject > m_compressions
Definition BinaryFile.h:498
String readData(uint04 max_size)
Definition BinaryFile.h:288
~BinaryFile()
Definition BinaryFile.h:57
Buffer< uint01, uint08 > m_compressed_data
Definition BinaryFile.h:496
uint04 readData(char *string, uint04 max_size)
Definition BinaryFile.h:296
Buffer< uint01, uint08 > m_cached_data
Definition BinaryFile.h:497
BinaryFile(const File &file)
Definition BinaryFile.h:54
void readString(String &string, char terminator='\0')
Definition BinaryFile.h:239
t_type read()
Definition BinaryFile.h:225
const Buffer< uint01, uint08 > & cachedData() const
Definition BinaryFile.h:109
A specification of upper and lower bounds in N-dimensions.
Definition Bounds.hpp:57
constexpr Ray< t_dims, t_type > span() const
The side lengths of these bounds. For each dimension, the span is max - min.
Definition Bounds.hpp:139
The equivelent of std::vector but with a bit more control. The basic array unit of the library.
Definition Buffer.hpp:64
constexpr t_index_type size() const
Definition Buffer.hpp:1461
decltype(auto) ptr()
Definition Buffer.hpp:489
void addAll(const Buffer< t_type, t_other_index_type, t_other_memory_allocator, t_other_memory_manager > &buffer)
Definition Buffer.hpp:248
void setSize(t_index_type new_size)
Definition Buffer.hpp:1413
decltype(auto) begin()
Definition Buffer.hpp:504
t_index_type memSize() const
Definition Buffer.hpp:499
void clear()
Definition Buffer.hpp:572
Definition Dictionary.h:48
Definition StringStream.h:62
@ MIN
Definition BaseValues.hpp:226
@ MAX
Definition BaseValues.hpp:227
float fltp04
Defines an alias representing a 4 byte floating-point number.
Definition BaseValues.hpp:157
uint8_t uint01
-Defines an alias representing a 1 byte, unsigned integer -Can represent exact integer values 0 throu...
Definition BaseValues.hpp:98
t_type ChangeEndian(t_type in)
Definition BinaryFile.h:43
CompressionMode
Definition Compressor.h:12
@ e_string_compression
Definition Compressor.h:16
@ e_string_reference
Definition Compressor.h:27
uint64_t uint08
-Defines an alias representing an 8 byte, unsigned integer -Can represent exact integer values 0 thro...
Definition BaseValues.hpp:132
uint32_t uint04
-Defines an alias representing a 4 byte, unsigned integer -Can represent exact integer values 0 throu...
Definition BaseValues.hpp:120
constexpr bool isNaN(const t_type &value)
Query if 'value' is valid or invalid.
Definition BaseFunctions.hpp:200
double fltp08
Defines an alias representing an 8 byte floating-point number.
Definition BaseValues.hpp:181
constexpr t_type getMin(const t_type &left, const t_type &right)
Finds the minimum of the given arguments based on the < operator.
Definition BaseFunctions.hpp:67
Definition Compressor.h:38
uint01 * compressed_data
Definition Compressor.h:42
uint08 compressed_size
Definition Compressor.h:43
uint01 * uncompressed_data
Definition Compressor.h:40
uint08 uncompressed_size
Definition Compressor.h:41
CompressionMode compression_mode
Definition Compressor.h:39
uint08 buffer_size
Definition Compressor.h:44
Definition BaseValues.hpp:272