TranslationTools.h

#pragma once
#include <NDEVR/Buffer.h>
#include <NDEVR/String.h>
#include <NDEVR/File.h>
#include <NDEVR/Scanner.h>
#include <NDEVR/INIFactory.h>
#include <NDEVR/ApplicationResource.h>
#include <NDEVR/Translator.h>
namespace NDEVR
{
    class TranslationTools
    {
    public:
        File translation_folder = File("$(NDEVR_SOURCE_DIR)\\Resources\\Translations\\");   
        Buffer<String> source_roots = { String("$(NDEVR_SOURCE_DIR)"), String("$(NDEVR_SOURCE_DIR)\\..\\Carlson\\") };  
        Buffer<String> extensions = { String(".h"), String(".hpp"), String(".cpp") };   
        Buffer<StringView> simple_tr = { "_t(\"" ,"_tq(\"","_tqa(\"","_tqs(\"", "_to(\"", "_ti(\"" };   
        Buffer<StringView> complex_tr = { "_td(\"" ,"_tdq(\"", "_tid(\"" }; 
        Buffer<StringView> complex_utf8_tr = { "_td(\"" ,"_tdq(\"","_tqo(\"" }; 
        Buffer<StringView> qt_tr = { "QApplication::translate(\"", "QCoreApplication::translate(\"" };  
        StringView utf8 = "utf8(\"";    
        void replaceStringVars(String& s)
        {
            s.replace("\\\\", "\\");
            s.replace("\\\"", "\"");
            s.replace("\\n", "\n");
            s.replace("\\", "\n");
            s.replace("\\t", "\t");
            s.replace("\\r", "\r");
        }

        void processString(String& s, Scanner& scan, uint04 start_index, const char* end_seq)
        {
            bool is_utf8 = s.beginsWith(utf8);
            if (is_utf8)
            {
                s = s.substr(utf8.size());
                start_index = 0;
            }
            uint04 end_index = s.indexOf(end_seq, '\\', false, start_index);
            if (s.size() > 0)
            {
                s = s.substr(start_index, end_index);
                replaceStringVars(s);
            }
            while (IsInvalid(end_index) && scan.nextLine())
            {
                s.trimWhiteSpace();// remove all whitespace
                if (s.size() > 0 && s.last() == '"')
                    s.removeLast();//remove end "
                String s2 = scan.currentLine();
                s2.trimWhiteSpace();
                if (s2.size() != 0)
                {
                    end_index = s2.indexOf(end_seq, false, start_index);
                    s += s2.substr(1, end_index);
                }
            }
            lib_assert(IsValid(end_index), "Unexpected end of file found");
        }

        void addOption(INIFactory& translator, const String& s_label, const String& value)
        {
            String label = Translator::DefaultReadableLabel(s_label);
 
#ifdef _DEBUG
            if (translator.hasOption(label))
            {
                String old_option = translator.getOption(label);
                lib_assert(old_option == value, "duplicate translation with differing definition");
            }
            else
#endif
            {
                translator.addManagedOption(label, value);
            }
        }

        void setupFile(INIFactory& translator, File file)
        {
            String translation;
            Scanner scan(file);
            while (scan.nextLine())
            {
                String s = scan.currentLine();
                for (uint04 i = 0; i < simple_tr.size(); i++)
                {
                    uint04 start_index = s.indexOf(simple_tr[i]);
                    if (IsValid(start_index))
                    {
                        processString(s, scan, start_index + simple_tr[i].size(), "\")");
                        addOption(translator, s, s);
 
                    }
                }
                for (uint04 i = 0; i < complex_tr.size(); i++)
                {
                    uint04 start_index = s.indexOf(complex_tr[i]);
                    if (IsValid(start_index))
                    {
                        uint04 end_index = s.indexOf("\"", false, start_index);
 
                        StringView label = s.substr(start_index + complex_tr[i].size(), end_index);
                        Buffer<String> parts = StringView::Convert(label.split(','));
                        parts[0].trimWhiteSpace();
                        parts[1].trimWhiteSpace();
                        if (parts.size() > 2)
                        {
                            while (!parts[0].contains('\"', 1))
                            {
                                parts[0] += "," + parts[1];
                                parts.removeIndex(1);
                            }
                            while (!parts[1].contains('\"', 1))
                            {
                                parts[1] += "," + parts[2];
                                parts.removeIndex(2);
                            }
                        }
                        parts[0].trimWhiteSpace();
                        parts[0].removeLast();
                        parts[1].trimWhiteSpace();
                        processString(parts[1], scan, 1, "\")");
                        addOption(translator, parts[0], parts[1]);
                    }
                }
                for (uint04 i = 0; i < qt_tr.size(); i++)
                {
                    uint04 start_index = s.indexOf(qt_tr[i]);
                    if (IsValid(start_index))
                    {
                        uint04 end_index = s.indexOf("\", nullptr", false, start_index);
 
                        StringView label = s.substr(start_index + qt_tr[i].size(), end_index);
                        Buffer<String> parts = StringView::Convert(label.split(','));
                        parts[0].trimWhiteSpace();
                        parts[1].trimWhiteSpace();
                        if (parts.size() > 2)
                        {
                            while (!parts[0].contains('\"', 1))
                            {
                                parts[0] += "," + parts[1];
                                parts.removeIndex(1);
                            }
                            while (parts[1].size() > 1 && !parts[1].contains('\"', 1) && parts.size() > 2)
                            {
                                parts[1] += "," + parts[2];
                                parts.removeIndex(2);
                            }
                        }
                        parts[0].trimWhiteSpace();
                        parts[0].removeLast();
                        parts[1].trimWhiteSpace();
                        replaceStringVars(parts[0]);
                        parts[1].trimWhiteSpace();
                        parts[1].removeIndex(0);
                        if (!parts[1].isSameNoCase("Form"))
                        {
                            replaceStringVars(parts[1]);
                            addOption(translator, Translator::DefaultReadableLabel(parts[0]) + "_" + Translator::DefaultReadableLabel(parts[1]), parts[1]);
                        }
                    }
                }
            }
        }

        void getFiles(Buffer<File>& files, const File& file)
        {
            if (file.isDirectory())
            {
                Buffer<File> children = file.getChildren();
                for (const File& f : children)
                {
                    getFiles(files, f);
                }
            }
            else
            {
                if (extensions.contains(file.getPath(File::e_file_extension)))
                {
                    files.add(file);
                }
            }
        }

        static void getDirs(Buffer<File>& files, const File& file)
        {
            if (file.isDirectory())
            {
                if (file.getFolderName() == "Resources" || file.getFolderName() == "Bin" || file.getFolderName() == "libraries" || file.getFolderName() == "ThirdParty")
                    return;
                files.add(file);
                Buffer<File> children = file.getChildren();
                for (const File& f : children)
                {
                    getDirs(files, f);
                }
            }
        }

        static void convertBrackets(INIFactory& ini)
        {
            for (auto iter = ini.extraOptionsRef().begin(); iter != ini.extraOptionsRef().end(); iter++)
            {
                uint04 start = iter.value()->indexOf('[');
                uint04 end = iter.value()->indexOf(']') + 1;
                while (IsValid(start) && IsValid(end) && start < end)
                {
                    StringView substring = iter.value()->substr(start, end);
                    iter.value()->replace(substring, "{" + String(substring.hash()) + "}");
                    start = iter.value()->indexOf('[');
                    end = iter.value()->indexOf(']') + 1;
                }
            }
        }

        static void convertFromBrackets(INIFactory& ini, const INIFactory& ref)
        {
            Dictionary<String, String*>& dictionary = ini.extraOptionsRef();
            Dictionary<String, String> reference = ref.extraOptions();
            for (auto iter = dictionary.begin(); iter != dictionary.end(); iter++)
            {
                if (!reference.contains(iter.key()))
                {
                    continue;
                }
                String other_val = reference.get(iter.key());
                uint04 o_start = other_val.indexOf('[');
                uint04 o_end = other_val.indexOf(']') + 1;
 
 
                while (IsValid(o_start) && IsValid(o_end) && o_start < o_end)
                {
                    StringView search_string = other_val.substr(o_start, o_end);
                    String replace_substring = "{" + String(search_string.hash()) + "}";
                    iter.value()->replace(replace_substring, search_string);
                    other_val.replace(search_string, replace_substring);
                    o_start = other_val.indexOf('[');
                    o_end = other_val.indexOf(']') + 1;
                }
            }
        }

        void makeTranslation()
        {
            ApplicationResource::application_name.set(_t("Translator"));
            Buffer<File> roots;
            for (const String& dir : source_roots)
            {
                File file(dir);
                file.expandEnvironmentalVars();
                getDirs(roots, file);
            }
 
            Buffer<File> files;
            for (File dir : roots)
            {
                if (dir.endsWith("\\Headers\\"))
                    getFiles(files, dir);
                if (dir.endsWith("\\Source\\"))
                    getFiles(files, dir);
                if (dir.endsWith("\\Generated\\"))
                    getFiles(files, dir);
            }
            File master(String(translation_folder) + "en_US.tr");
            File hash_master(String(translation_folder) + "en_US.hash");
            INIFactory root_ini;
            root_ini.setPreserveOrder(true);
            for (const File& f : files)
            {
                setupFile(root_ini, f);
            }
            root_ini.writeToAsciiFile(master);
 
 
            Buffer<File> other_translations = translation_folder.getChildren();
            for (uint04 i = 0; i < other_translations.size(); i++)
            {
                if (other_translations[i].getPath(File::e_file_extension).isSameNoCase(".tr")
                    || other_translations[i].getPath(File::e_file_extension).isSameNoCase(".hash"))
                {
                    if (other_translations[i] != master && other_translations[i] != hash_master)
                    {
                        INIFactory ini;
                        ini.setPreserveOrder(true);
                        ini.readAsciiFile(master);
                        if (other_translations[i].endsWith(".hash"))
                            ini.setUseHashLabels(true);
                        ini.readAsciiFile(other_translations[i]);
                        if (other_translations[i].endsWith(".hash"))
                        {
                            ini.setUseHashLabels(false);
                            convertFromBrackets(ini, root_ini);
                            File f = other_translations[i];
                            f.setPath(".tr", File::e_file_extension);
                            if (f.exists())
                                ini.readAsciiFile(f);
                            ini.writeToAsciiFile(f);
                        }
                        else
                        {
                            ini.writeToAsciiFile(other_translations[i]);
                        }
                    }
                }
            }
            convertBrackets(root_ini);
            root_ini.setUseHashLabels(true);
            root_ini.writeToAsciiFile(hash_master);//write hash translation for easy web translation
        }
    };
}