token_stream.h

Go to the documentation of this file.

// MIT License
//
// Copyright (c) 2019 Ruhr University Bochum, Chair for Embedded Security. All Rights reserved.
// Copyright (c) 2019 Marc Fyrbiak, Sebastian Wallat, Max Hoffmann ("ORIGINAL AUTHORS"). All rights reserved.
// Copyright (c) 2021 Max Planck Institute for Security and Privacy. All Rights reserved.
// Copyright (c) 2021 Jörn Langheinrich, Julian Speith, Nils Albartus, René Walendy, Simon Klix ("ORIGINAL AUTHORS"). All Rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
 
#pragma once
 
#include "hal_core/defines.h"
 
#include <algorithm>
#include <map>
#include <vector>
 
namespace hal
{
    template<typename T>
    struct CORE_API Token
    {
        Token(u32 n, const T s) : number(n), string(s)
        {
        }
 
        u32 number;
 
        T string;
 
        operator T() const
        {
            return string;
        }
 
        Token<T>& operator=(const T& s)
        {
            this->string = s;
            return *this;
        }
 
        Token<T>& operator+=(const T& s)
        {
            this->string += s;
            return *this;
        }
 
        bool operator==(const T& s) const
        {
            return string == s;
        }
 
        bool operator!=(const T& s) const
        {
            return !(*this == s);
        }
    };
 
    template<typename T>
    class NETLIST_API TokenStream
    {
    public:
        struct TokenStreamException
        {
            T message;
 
            u32 line_number;
        };
 
        static const u32 END_OF_STREAM = 0xFFFFFFFF;
 
        TokenStream(const std::vector<T>& increase_level_tokens = {"("}, const std::vector<T>& decrease_level_tokens = {")"})
        {
            m_pos                   = 0;
            m_increase_level_tokens = increase_level_tokens;
            m_decrease_level_tokens = decrease_level_tokens;
        }
 
        TokenStream(const std::vector<Token<T>>& init, const std::vector<T>& increase_level_tokens = {"("}, const std::vector<T>& decrease_level_tokens = {")"})
            : TokenStream(increase_level_tokens, decrease_level_tokens)
        {
            m_data = init;
        }
 
        TokenStream(const TokenStream<T>& other)
        {
            m_pos                   = other.m_pos;
            m_data                  = other.m_data;
            m_increase_level_tokens = other.m_increase_level_tokens;
            m_decrease_level_tokens = other.m_decrease_level_tokens;
        }
 
        TokenStream<T>& operator=(const TokenStream<T>& other)
        {
            m_pos                   = other.m_pos;
            m_data                  = other.m_data;
            m_increase_level_tokens = other.m_increase_level_tokens;
            m_decrease_level_tokens = other.m_decrease_level_tokens;
            return *this;
        }
 
        Token<T> consume(u32 num = 1)
        {
            m_pos += num;
            return at(m_pos - 1);
        }
 
        bool consume(const T& expected, bool throw_on_error = false)
        {
            if (m_pos >= size())
            {
                if (throw_on_error)
                {
                    throw TokenStreamException({"expected Token '" + expected + "' but reached the end of the stream", get_current_line_number()});
                }
                return false;
            }
 
            if (at(m_pos) != expected)
            {
                if (throw_on_error)
                {
                    throw TokenStreamException({"expected Token '" + expected + "' but got '" + at(m_pos).string + "'", get_current_line_number()});
                }
                return false;
            }
 
            m_pos++;
            return true;
        }
 
        Token<T> consume_until(const T& expected, u32 end = END_OF_STREAM, bool level_aware = true, bool throw_on_error = false)
        {
            auto found = find_next(expected, end, level_aware);
            if (found > size() && throw_on_error)
            {
                throw TokenStreamException({"expected Token '" + expected + "' not found", get_current_line_number()});
            }
            m_pos = std::min(size(), found);
            return at(m_pos - 1);
        }
 
        Token<T> consume_current_line()
        {
            u32 line = at(m_pos).number;
 
            u32 i = m_pos;
            while (++i < size() && at(i).number == line)
            {
            }
            m_pos = i;
 
            return at(m_pos - 1);
        }
 
        TokenStream<T> extract_until(const T& expected, u32 end = END_OF_STREAM, bool level_aware = true, bool throw_on_error = false)
        {
            auto found = find_next(expected, end, level_aware);
            if (found > size() && throw_on_error)
            {
                throw TokenStreamException({"expected Token '" + expected + "' not found", get_current_line_number()});
            }
            auto end_pos = std::min(size(), found);
            TokenStream res(m_increase_level_tokens, m_decrease_level_tokens);
            res.m_data.reserve(res.m_data.size() + (end_pos - m_pos));
            res.m_data.insert(res.m_data.begin(), m_data.begin() + m_pos, m_data.begin() + end_pos);
            m_pos = end_pos;
            return res;
        }
 
        Token<T> join_until(const T& match, const T& joiner, u32 end = END_OF_STREAM, bool level_aware = true, bool throw_on_error = false)
        {
            u32 start_line = get_current_line_number();
            auto found     = find_next(match, end, level_aware);
            if (found > size() && throw_on_error)
            {
                throw TokenStreamException({"match Token '" + match + "' not found", start_line});
            }
            auto end_pos = std::min(size(), found);
            T result;
            while (m_pos < end_pos && remaining() > 0)
            {
                if (!result.empty())
                {
                    result += joiner;
                }
                result += consume();
            }
            return {start_line, result};
        }
 
        Token<T> join(const T& joiner)
        {
            u32 start_line = get_current_line_number();
            T result;
            while (remaining() > 0)
            {
                if (!result.empty())
                {
                    result += joiner;
                }
                result += consume();
            }
            return {start_line, result};
        }
 
        Token<T>& peek(i32 offset = 0)
        {
            return at(m_pos + offset);
        }
 
        const Token<T>& peek(i32 offset = 0) const
        {
            return at(m_pos + offset);
        }
 
        Token<T>& at(u32 position)
        {
            if (position >= m_data.size())
            {
                throw TokenStreamException({"reached the end of the stream", get_current_line_number()});
            }
            return m_data[position];
        }
 
        const Token<T>& at(u32 position) const
        {
            if (position >= m_data.size())
            {
                throw TokenStreamException({"reached the end of the stream", get_current_line_number()});
            }
            return m_data[position];
        }
 
        u32 find_next(const T& match, u32 end = END_OF_STREAM, bool level_aware = true) const
        {
            u32 level = 0;
            for (u32 i = m_pos; i < size() && i < end; ++i)
            {
                const auto& Token = at(i);
                if ((!level_aware || level == 0) && Token == match)
                {
                    return i;
                }
                else if (level_aware && std::find_if(m_increase_level_tokens.begin(), m_increase_level_tokens.end(), [&Token](const auto& x) { return Token == x; }) != m_increase_level_tokens.end())
                {
                    level++;
                }
                else if (level_aware && level > 0
                         && std::find_if(m_decrease_level_tokens.begin(), m_decrease_level_tokens.end(), [&Token](const auto& x) { return Token == x; }) != m_decrease_level_tokens.end())
                {
                    level--;
                }
            }
            return end;
        }
 
        u32 size() const
        {
            return m_data.size();
        }
 
        u32 consumed() const
        {
            return m_pos;
        }
 
        u32 remaining() const
        {
            return size() - consumed();
        }
 
        u32 position() const
        {
            return m_pos;
        }
 
        void set_position(u32 p)
        {
            m_pos = p;
        }
 
    private:
        std::vector<T> m_increase_level_tokens;
        std::vector<T> m_decrease_level_tokens;
        std::vector<Token<T>> m_data;
        u32 m_pos;
 
        u32 get_current_line_number() const
        {
            if (m_pos < m_data.size())
            {
                return m_data[m_pos].number;
            }
            else if (!m_data.empty())
            {
                return m_data.back().number;
            }
            return END_OF_STREAM;
        }
    };
}    // namespace hal