API Documentation
Loading...
Searching...
No Matches
BinaryFile.h
Go to the documentation of this file.
1/**--------------------------------------------------------------------------------------------
2Copyright (c) 2019, NDEVR LLC
3tyler.parke@ndevr.org
4 __ __ ____ _____ __ __ _______
5 | \ | | | __ \ | ___|\ \ / / | __ \
6 | \ | | | | \ \ | |___ \ \ / / | |__) |
7 | . \| | | |__/ / | |___ \ V / | _ /
8 | |\ |_|_____/__|_____|___\_/____| | \ \
9 |__| \__________________________________| \__\
10
11Subject to the terms of the Enterprise+ Agreement, NDEVR hereby grants
12Licensee a limited, non-exclusive, non-transferable, royalty-free license
13(without the right to sublicense) to use the API solely for the purpose of
14Licensee's internal development efforts to develop applications for which
15the API was provided.
16
17The above copyright notice and this permission notice shall be included in all
18copies or substantial portions of the Software.
19
20THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
21INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
22PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
23FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
25DEALINGS IN THE SOFTWARE.
26
27Library: Base
28File: BinaryFile
29Included in API: True
30Author(s): Tyler Parke
31 *-----------------------------------------------------------------------------------------**/
32#pragma once
33#include "DLLInfo.h"
34#include <NDEVR/File.h>
35#include <NDEVR/String.h>
36#include <NDEVR/Dictionary.h>
37#include <NDEVR/Bounds.h>
38#include <NDEVR/Compressor.h>
39#include <fstream>
40namespace NDEVR
41{
42 template <class t_type>
43 t_type ChangeEndian(t_type in)
44 {
45 char* const p = reinterpret_cast<char*>(&in);
46 for (size_t i = 0; i < sizeof(t_type) / 2; ++i)
47 std::swap(p[i], p[sizeof(t_type) - i - 1]);
48 return in;
49 }
51 {
52 public:
53
54 explicit BinaryFile(const File& file)
55 : m_file(file)
56 {}
58 {
59 close();
60 }
61 void open(bool read, bool safe)
62 {
63 m_is_read = read;
64 if (read)
65 m_file.open(safe ? File::e_binary_read_safe : File::e_binary_read);
66 else
67 m_file.open(safe ? File::e_binary_write_safe : File::e_binary_write);
68 }
69 void cachedOpen(bool read)
70 {
71 m_is_read = read;
72 m_using_cache = true;
73 m_cache_location = 0;
74 if (read)
75 {
76 m_file.open(File::e_binary_read);
77 m_file.invalidateCache();
78 uint08 size = m_file.fileSize();
79 m_cached_data.setSize(size);
80 fread(m_cached_data.begin(), 1, m_cached_data.size(), m_file.filePtr());
81 m_file.close();
82 }
83 else
84 {
85 m_cached_data.clear();
86 }
87
88 }
89 void close()
90 {
91 if (m_is_read)
92 {
93 if (!m_using_cache)
94 m_file.close();
95 else if (m_multitreaded_compression)
96 uncompressAll();
97 }
98 else
99 {
100 if (m_using_cache)
101 {
102 m_file.open(File::e_binary_write);
103 fwrite(m_cached_data.begin(), 1, m_cached_data.size(), m_file.filePtr());
104 m_file.close();
105 }
106 }
107
108 }
110 {
111 return m_cached_data;
112 }
113 //Data writing
114 template<class t_type>
115 void write(const t_type& data)
116 {
117 if (m_using_cache)
118 {
119
120 if (m_cache_location == m_cached_data.size())
121 m_cached_data.addAll((uint01*)&data, sizeof(data));
122 else
123 m_cached_data.setAll((uint01*)&data, m_cache_location, sizeof(data));
124 m_cache_location += sizeof(data);
125 }
126 else
127 {
128 fwrite(&data, sizeof(t_type), 1, m_file.filePtr());
129 }
130
131 }
132
133 template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
134 typename std::enable_if<!ObjectInfo<t_type>::Buffer>::type write(const Buffer<t_type, t_index_type, t_memory_allocator, t_memory_manager>& data, CompressionMode compression_mode)
135 {
136 if (data.size() != 0)
137 {
138 BinaryCompressionObject binary = createCompressionObject();
139 binary.buffer_size = data.size();
140 binary.compression_mode = Compressor::PickCompressionMode<t_type>(compression_mode);
141 binary.uncompressed_data = (uint01*)data.ptr();
142 binary.uncompressed_size = data.memSize();
144 {
145 m_compressed_data.setSize(sizeof(Bounds<1, uint04>));
146 binary.compressed_size = sizeof(Bounds<1, uint04>);
147 }
148 else if (binary.compression_mode == e_string_reference)
149 {
150 m_compressed_data.clear();
151 binary.compressed_size = 0;
152 }
153 else
154 {
155 m_compressed_data.setSize(data.memSize());
156 binary.compressed_size = data.memSize();
157 }
158 binary.compressed_data = m_compressed_data.begin();
159 Compressor::CompressData(binary);
160 writeCompression(binary);
161 }
162 else
163 {
164 //lib_assert(m_large_compression || cast<uint08>(data.size()) < cast<uint08>(cast<uint04>::Max), "Bad Data write");
165 if (m_large_compression)
166 write(cast<uint08>(data.size()));
167 else
168 write(cast<uint04>(data.size()));
169 }
170 }
171 template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
172 typename std::enable_if<ObjectInfo<t_type>::Buffer>::type write(const Buffer<t_type, t_index_type, t_memory_allocator, t_memory_manager>& data, CompressionMode compression_mode)
173 {
174 uint04 total_size = 0;
175 Buffer<uint04> sizes;
176 sizes.setSize(data.size());
177 for (uint04 i = 0; i < data.size(); i++)
178 {
179 uint04 size = data[i].size();;
180 sizes[i] = size;
181 total_size += size;
182 }
183 Buffer<decltype(t_type::Type())> combined_data(total_size);
184 for (uint04 i = 0; i < data.size(); i++)
185 {
186 combined_data.addAll(data[i]);
187 }
188 write(sizes, compression_mode);
189 write(combined_data, compression_mode);
190 }
191 void writeRawData(const String& data, uint04 size, char fill_space = '\0')
192 {
193 if (m_using_cache)
194 {
195
196 if (m_cache_location == m_cached_data.size())
197 m_cached_data.addAll((uint01*)data.begin(), getMin(data.size(), size));
198 else
199 m_cached_data.setAll((uint01*)data.begin(), m_cache_location, getMin(data.size(), size));
200 m_cache_location += getMin(data.size(), size);
201 if (size > data.size())
202 {
203 uint04 add_size = size - data.size();
204 if (m_cache_location == m_cached_data.size())
205 m_cached_data.setAllToValue(fill_space, m_cache_location, add_size);
206 else
207 m_cached_data.addAndFillSpace(add_size, fill_space);
208 m_cache_location += add_size;
209 }
210
211 }
212 else
213 {
214 fwrite(data.begin(), data.size(), 1, m_file.filePtr());
215 if (size > data.size())
216 {
217 uint04 add_size = size - data.size();
218 for(uint04 i = 0; i < add_size; i++)
219 fwrite(&fill_space, add_size, 1, m_file.filePtr());
220 }
221 }
222 }
223 //data reading
224 template<class t_type>
225 t_type read()
226 {
227 t_type data;
228 if (m_using_cache)
229 {
230 data = *((t_type*)(&m_cached_data[m_cache_location]));
231 m_cache_location += cast<uint08>(sizeof(t_type));
232 }
233 else
234 {
235 fread(&data, sizeof(t_type), 1, m_file.filePtr());
236 }
237 return data;
238 }
239 void readString(String& string, char terminator = '\0')
240 {
241 if (m_using_cache)
242 {
243 for (;;)
244 {
245 char c = *((char*)(&m_cached_data[m_cache_location++]));
246 if (c == terminator || m_cache_location >= m_cached_data.size())
247 return;
248 string.add(c);
249 }
250 }
251 else
252 {
253 for (;;)
254 {
255 char c = cast<char>(fgetc(m_file.filePtr()));
256 if (c == terminator || feof(m_file.filePtr()))
257 return;
258 string.add(c);
259 }
260 }
261 }
262 uint04 readString(char* string, char terminator = '\0')
263 {
264 uint04 size = 0;
265 if (m_using_cache)
266 {
267 for (;;)
268 {
269 char c = *((char*)(&m_cached_data[m_cache_location++]));
270 if (c == terminator || m_cache_location >= m_cached_data.size())
271 break;
272 string[size++] = c;
273 }
274 }
275 else
276 {
277 for (;;)
278 {
279 char c = cast<char>(fgetc(m_file.filePtr()));
280 if (c == terminator || feof(m_file.filePtr()))
281 break;
282 string[size++] = c;
283 }
284 }
285 string[size] = '\0';
286 return size;
287 }
289 {
290 String string;
291 string.setSize(max_size);
292 uint04 actual_size = readData(string.begin(), max_size);
293 string.setSize(actual_size);
294 return string;
295 }
296 uint04 readData(char* string, uint04 max_size)
297 {
298 if (m_using_cache)
299 {
300 for (uint04 i = 0; i < max_size; i++)
301 {
302 char c = *((char*)(&m_cached_data[m_cache_location++]));
303 if (m_cache_location >= m_cached_data.size())
304 return i;
305 string[i] = c;
306 }
307 }
308 else
309 {
310 for (uint04 i = 0; i < max_size; i++)
311 {
312 char c = cast<char>(fgetc(m_file.filePtr()));
313 if (feof(m_file.filePtr()))
314 return i;
315 string[i] = c;
316 }
317 }
318 return max_size;
319 }
320 template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
322 {
323 bool allow_multithread = m_multitreaded_compression;
324 read(data, allow_multithread);
325 }
326 template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
328 {
329 bool allow_multithread = false;
330 read(data, allow_multithread);
331 }
332
333 template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
334 typename std::enable_if<ObjectInfo<t_type>::Buffer>::type read(Buffer<t_type, t_index_type, t_memory_allocator, t_memory_manager>& data, bool& allow_multithread)
335 {
336 Buffer<uint04> sizes;
337 Buffer<decltype(t_type::Type())> combined_data;
338 //if (ObjectInfo<t_type>::String)
339 allow_multithread = false;
340 read(sizes, allow_multithread);
341 read(combined_data, allow_multithread);
342 data.setSize(sizes.size());
343 uint04 current_offset = 0;
344 for (uint04 i = 0; i < sizes.size(); i++)
345 {
346 data[i].clear();
347 if (sizes[i] > 0)
348 data[i].addAll(&combined_data[0] + current_offset, sizes[i]);
349 current_offset += sizes[i];
350 }
351 }
352
353 template<class t_index_type, class t_memory_allocator, class t_memory_manager>
355 {
356 Buffer<uint04> indices;
357 //if (ObjectInfo<t_type>::String)
358 allow_multithread = false;
359 read(indices, allow_multithread);
360 data.setSize(indices.size());
361 for (uint04 i = 0; i < indices.size(); i++)
362 {
363 Bounds<1, uint04> bounds = m_string_size_info[indices[i]];
364 if (bounds.span() > 0U)
365 {
366 data[i] = m_strings.substr(bounds[MIN], bounds[MAX]);
367 }
368 else
369 {
370 data[i].clear();
371 }
372 }
373 }
374
375 template<class t_type, class t_index_type, class t_memory_allocator, class t_memory_manager>
376 typename std::enable_if<!ObjectInfo<t_type>::Buffer>::type read(Buffer<t_type, t_index_type, t_memory_allocator, t_memory_manager>& data, bool& allow_multithread)
377 {
378 BinaryCompressionObject binary = createCompressionObject();
379 if (m_large_compression)
380 binary.uncompressed_size = read<uint08>();
381 else
382 binary.uncompressed_size = read<uint04>();
383 //lib_assert(binary.uncompressed_size < Constant<sint04>::Max, "Large number read");
384 data.setSize(cast<t_index_type>(binary.uncompressed_size));
385 binary.uncompressed_size = data.memSize();
386 if (data.size() != 0)
387 {
388 binary.uncompressed_data = (uint01*)data.ptr();
389 if (m_large_compression)
390 binary.compressed_size = read<uint08>();
391 else
392 binary.compressed_size = read<uint04>();
393 if (m_using_cache)
394 {
395 binary.compression_mode = cast<CompressionMode>(m_cached_data[m_cache_location]);
396 binary.compressed_size -= 1;
397 if (binary.compression_mode != CompressionMode::e_string_reference)
398 {
399 binary.compressed_data = &m_cached_data[m_cache_location + 1];
400
401 allow_multithread &= binary.compression_mode != CompressionMode::e_string_compression;
402 allow_multithread &= binary.compression_mode != CompressionMode::e_no_compression;
403 if (m_multitreaded_compression && allow_multithread)
404 m_compressions.add(binary);
405 else
406 Compressor::DecompressData(binary);
407 m_cache_location += binary.compressed_size + 1;
408 }
409 else
410 {
411 Compressor::DecompressData(binary);
412 m_cache_location++;
413 }
414 }
415 else
416 {
417 if (binary.compression_mode != CompressionMode::e_string_reference)
418 {
419 m_compressed_data.setSize(binary.compressed_size);
420#ifdef USE_ASSERTIONS
421 size_t fsize = fread(&m_compressed_data[0], 1, m_compressed_data.size(), m_file.filePtr());
422 lib_assert(m_compressed_data.size() == cast<uint08>(fsize), "Bad read size");
423#else
424 fread(&m_compressed_data[0], 1, m_compressed_data.size(), m_file.filePtr());
425#endif
426 binary.compression_mode = cast<CompressionMode>(m_compressed_data[0]);
427 binary.compressed_data = &m_compressed_data[1];
428 binary.compressed_size -= 1;
429 Compressor::DecompressData(binary);
430 m_cache_location += binary.compressed_size + 1;
431 }
432 else
433 {
434 Compressor::DecompressData(binary);
435 m_cache_location++;
436 }
437 }
438 }
439 }
440#ifdef _WIN32
441#define ndevr_ftell _ftelli64
442#define ndevr_fseek _fseeki64
443#else
444#define ndevr_ftell ftell
445#define ndevr_fseek fseek
446#endif
448 {
449 if (m_using_cache)
450 return m_cache_location;
451 else
452 return cast<uint08>(ndevr_ftell(m_file.filePtr()));
453 }
455 {
456 if (m_using_cache)
457 {
458 return cast<fltp04>(m_cache_location) / cast<fltp04>(m_cached_data.size());
459 }
460 else
461 {
462 if (isNaN(m_file_size))
463 m_file_size = m_file.fileSize();
464 return cast<fltp04>(cast<uint08>(ndevr_ftell(m_file.filePtr()))) / cast<fltp04>(m_file_size);
465 }
466 }
467 void seek(uint08 location)
468 {
469 if (m_using_cache)
470 m_cache_location = location;
471 else
472 ndevr_fseek(m_file.filePtr(), cast<long long>(location), SEEK_SET);
473 }
474 BinaryCompressionObject createCompressionObject();
475 void setUseLargeCompression(bool use_large_compression) { m_large_compression = use_large_compression; }
476 bool isFinished();
477 void writeCompression(BinaryCompressionObject& compression_object);
478 void writeStringData(CompressionMode compression_mode);
479 void readStringData(bool has_reference_table);
480 void uncompressAll();
481 void uncompressSections(uint04 start, uint04 count);
483 {
484 if (m_using_cache)
485 return m_cached_data.size();
486 if (isNaN(m_file_size))
487 m_file_size = m_file.fileSize();
488 return m_file_size;
489 }
490 protected:
499 uint08 m_cache_location = Constant<uint08>::NaN;
500 mutable uint08 m_file_size = Constant<uint08>::NaN;
501 bool m_using_cache = false;
502 bool m_is_read = false;
503 bool m_large_compression = false;
504 bool m_multitreaded_compression = true;
505 };
507}
#define NDEVR_BASE_API
Definition DLLInfo.h:78
#define ndevr_fseek
Definition BinaryFile.h:445
#define ndevr_ftell
Definition BinaryFile.h:444
#define lib_assert(expression, message)
Asserts some logic in the code. Disabled in non debug mode by default. Can be re-enabled in release u...
Definition LibAssert.h:70
Definition BinaryFile.h:51
uint04 readString(char *string, char terminator='\0')
Definition BinaryFile.h:262
void read(Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data)
Definition BinaryFile.h:321
void setUseLargeCompression(bool use_large_compression)
Definition BinaryFile.h:475
std::enable_if<!ObjectInfo< t_type >::Buffer >::type read(Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data, bool &allow_multithread)
Definition BinaryFile.h:376
Buffer< Bounds< 1, uint04 > > m_string_size_info
Definition BinaryFile.h:494
void write(const t_type &data)
Definition BinaryFile.h:115
fltp04 percent()
Definition BinaryFile.h:454
std::enable_if< ObjectInfo< t_type >::Buffer >::type write(const Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data, CompressionMode compression_mode)
Definition BinaryFile.h:172
void seek(uint08 location)
Definition BinaryFile.h:467
std::enable_if<!ObjectInfo< t_type >::Buffer >::type write(const Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data, CompressionMode compression_mode)
Definition BinaryFile.h:134
void close()
Definition BinaryFile.h:89
void open(bool read, bool safe)
Definition BinaryFile.h:61
Dictionary< String, uint04 > m_string_reference
Definition BinaryFile.h:493
String m_strings
Definition BinaryFile.h:492
void cachedOpen(bool read)
Definition BinaryFile.h:69
std::enable_if< ObjectInfo< t_type >::Buffer >::type read(Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data, bool &allow_multithread)
Definition BinaryFile.h:334
void readNow(Buffer< t_type, t_index_type, t_memory_allocator, t_memory_manager > &data)
Definition BinaryFile.h:327
uint08 fileSize() const
Definition BinaryFile.h:482
void writeRawData(const String &data, uint04 size, char fill_space='\0')
Definition BinaryFile.h:191
uint08 position()
Definition BinaryFile.h:447
void readStringBuffer(Buffer< String, t_index_type, t_memory_allocator, t_memory_manager > &data, bool &allow_multithread)
Definition BinaryFile.h:354
File m_file
Definition BinaryFile.h:491
Buffer< BinaryCompressionObject > m_compressions
Definition BinaryFile.h:498
String readData(uint04 max_size)
Definition BinaryFile.h:288
~BinaryFile()
Definition BinaryFile.h:57
Buffer< uint01, uint08 > m_compressed_data
Definition BinaryFile.h:496
uint04 readData(char *string, uint04 max_size)
Definition BinaryFile.h:296
Buffer< uint01, uint08 > m_cached_data
Definition BinaryFile.h:497
BinaryFile(const File &file)
Definition BinaryFile.h:54
void readString(String &string, char terminator='\0')
Definition BinaryFile.h:239
t_type read()
Definition BinaryFile.h:225
const Buffer< uint01, uint08 > & cachedData() const
Definition BinaryFile.h:109
A specification of upper and lower bounds in N-dimensions.
Definition Bounds.hpp:57
constexpr Ray< t_dims, t_type > span() const
The side lengths of these bounds. For each dimension, the span is max - min.
Definition Bounds.hpp:139
The equivelent of std::vector but with a bit more control. The basic array unit of the library.
Definition Buffer.hpp:64
constexpr t_index_type size() const
Definition Buffer.hpp:1461
decltype(auto) ptr()
Definition Buffer.hpp:489
void addAll(const Buffer< t_type, t_other_index_type, t_other_memory_allocator, t_other_memory_manager > &buffer)
Definition Buffer.hpp:248
void setSize(t_index_type new_size)
Definition Buffer.hpp:1413
decltype(auto) begin()
Definition Buffer.hpp:504
t_index_type memSize() const
Definition Buffer.hpp:499
void clear()
Definition Buffer.hpp:572
Definition Dictionary.h:48
Definition File.h:47
Definition String.h:40
Definition StringStream.h:62
Definition ACIColor.h:37
@ MIN
Definition BaseValues.hpp:226
@ MAX
Definition BaseValues.hpp:227
float fltp04
Defines an alias representing a 4 byte floating-point number.
Definition BaseValues.hpp:157
uint8_t uint01
-Defines an alias representing a 1 byte, unsigned integer -Can represent exact integer values 0 throu...
Definition BaseValues.hpp:98
t_type ChangeEndian(t_type in)
Definition BinaryFile.h:43
CompressionMode
Definition Compressor.h:12
@ e_string_compression
Definition Compressor.h:16
@ e_string_reference
Definition Compressor.h:27
uint64_t uint08
-Defines an alias representing an 8 byte, unsigned integer -Can represent exact integer values 0 thro...
Definition BaseValues.hpp:132
uint32_t uint04
-Defines an alias representing a 4 byte, unsigned integer -Can represent exact integer values 0 throu...
Definition BaseValues.hpp:120
constexpr bool isNaN(const t_type &value)
Query if 'value' is valid or invalid.
Definition BaseFunctions.hpp:200
double fltp08
Defines an alias representing an 8 byte floating-point number.
Definition BaseValues.hpp:181
constexpr t_type getMin(const t_type &left, const t_type &right)
Finds the minimum of the given arguments based on the < operator.
Definition BaseFunctions.hpp:67
Definition Compressor.h:38
uint01 * compressed_data
Definition Compressor.h:42
uint08 compressed_size
Definition Compressor.h:43
uint01 * uncompressed_data
Definition Compressor.h:40
uint08 uncompressed_size
Definition Compressor.h:41
CompressionMode compression_mode
Definition Compressor.h:39
uint08 buffer_size
Definition Compressor.h:44
Definition BaseValues.hpp:272