#include <token_stream.h>

Collaboration diagram for hal::TokenStream< T >:

Classes
struct	TokenStreamException

Public Member Functions
	TokenStream (const std::vector< T > &increase_level_tokens={"("}, const std::vector< T > &decrease_level_tokens={")"})

	TokenStream (const std::vector< Token< T >> &init, const std::vector< T > &increase_level_tokens={"("}, const std::vector< T > &decrease_level_tokens={")"})

	TokenStream (const TokenStream< T > &other)

TokenStream< T > &	operator= (const TokenStream< T > &other)

Token< T >	consume (u32 num=1)

bool	consume (const T &expected, bool throw_on_error=false)

Token< T >	consume_until (const T &expected, u32 end=END_OF_STREAM, bool level_aware=true, bool throw_on_error=false)

Token< T >	consume_current_line ()

TokenStream< T >	extract_until (const T &expected, u32 end=END_OF_STREAM, bool level_aware=true, bool throw_on_error=false)

Token< T >	join_until (const T &match, const T &joiner, u32 end=END_OF_STREAM, bool level_aware=true, bool throw_on_error=false)

Token< T >	join (const T &joiner)

Token< T > &	peek (i32 offset=0)

const Token< T > &	peek (i32 offset=0) const

Token< T > &	at (u32 position)

const Token< T > &	at (u32 position) const

u32	find_next (const T &match, u32 end=END_OF_STREAM, bool level_aware=true) const

u32	size () const

u32	consumed () const

u32	remaining () const

u32	position () const

void	set_position (u32 p)

Static Public Attributes
static const u32	END_OF_STREAM = 0xFFFFFFFF

Detailed Description

template<typename T>
class hal::TokenStream< T >

A token stream comprises a sequence of tokens that may, for example, have been read from a file.

Definition at line 125 of file token_stream.h.

Constructor & Destructor Documentation

◆ TokenStream() [1/3]

template<typename T >

hal::TokenStream< T >::TokenStream	(	const std::vector< T > &	increase_level_tokens = `{"("}`,
		const std::vector< T > &	decrease_level_tokens = `{")"}`
	)

inline

Construct an empty token stream.
The increase-level and decrease-level tokens are used for level-aware iteration. If active, all operations are only executed on tokens on level 0.
Example: consuming until "b" in 'a,(,b,),b,c' would consume 'a,(,b,)', if "(" and ")" are level increase/decrease tokens.

Parameters

[in]	decrease_level_tokens	- A vector of tokens that mark the start of a new level, i.e., increase the level.
[in]	increase_level_tokens	- A vector of tokens that mark the end of a level, i.e., decrease the level.

Definition at line 157 of file token_stream.h.

◆ TokenStream() [2/3]

template<typename T >

hal::TokenStream< T >::TokenStream	(	const std::vector< Token< T >> &	init,
		const std::vector< T > &	increase_level_tokens = `{"("}`,
		const std::vector< T > &	decrease_level_tokens = `{")"}`
	)

inline

Construct a token stream from a vector of tokens.
The increase-level and decrease-level tokens are used for level-aware iteration. If active, all operations are only executed on tokens on level 0.
Example: consuming until "b" in 'a,(,b,),b,c' would consume 'a,(,b,)', if "(" and ")" are level increase/decrease tokens.

Parameters

[in]	init	- The vector of tokens.
[in]	decrease_level_tokens	- A vector of tokens that mark the start of a new level, i.e., increase the level.
[in]	increase_level_tokens	- A vector of tokens that mark the end of a level, i.e., decrease the level.

Definition at line 173 of file token_stream.h.

◆ TokenStream() [3/3]

template<typename T >

hal::TokenStream< T >::TokenStream ( const TokenStream< T > & other )

inline

Construct a token stream from another one (i.e., copy constructor).

Parameters

[in] other - The token stream to copy.

Definition at line 184 of file token_stream.h.

Member Function Documentation

◆ at() [1/2]

template<typename T >

Token<T>& hal::TokenStream< T >::at ( u32 position )

inline

Return a Token at an absolute position in the stream without consuming it.
Can also access tokens that have already been consumed.
A position of 0 referse to the first element of the stream.

Throws a std::out_of_range exception if the position is not contained in the stream.

Parameters

[in] position - The absolute position in the stream.

Returns: The token at the queried position.

Definition at line 415 of file token_stream.h.

◆ at() [2/2]

template<typename T >

const Token<T>& hal::TokenStream< T >::at ( u32 position ) const

inline

Return a Token at an absolute position in the stream without consuming it.
Can also access tokens that have already been consumed.
A position of 0 referse to the first element of the stream.

Throws a std::out_of_range exception if the position is not contained in the stream.

Parameters

[in] position - The absolute position in the stream.

Returns: The token at the queried position.

Definition at line 427 of file token_stream.h.

◆ consume() [1/2]

template<typename T >

bool hal::TokenStream< T >::consume	(	const T &	expected,
		bool	throw_on_error = `false`
	)

inline

Consume the next token in the stream if it matches the given expected string.
If the token does not match, the stream remains unchanged.

Parameters

[in]	expected	- The expected token.
[in]	throw_on_error	- If true, throws an TokenStreamException instead of returning false on error.

Returns: True if the next Token matches the expected string, false otherwise or if no more tokens are available.

Definition at line 230 of file token_stream.h.

◆ consume() [2/2]

template<typename T >

Token<T> hal::TokenStream< T >::consume ( u32 num = 1 )

inline

Consume the next token(s) in the stream.
Advances the stream by the given number and returns the last consumed token.

Throws a std::out_of_range exception if more tokens are consumed than are available.

Parameters

[in] num - The amount of tokens to consume.

Returns: The last consumed token.

Definition at line 216 of file token_stream.h.

◆ consume_current_line()

template<typename T >

Token<T> hal::TokenStream< T >::consume_current_line ( )

inline

Consume the entire current line and returns the last consumed token.

Returns: The last consumed token or the last token of the stream.

Definition at line 282 of file token_stream.h.

◆ consume_until()

template<typename T >

Token<T> hal::TokenStream< T >::consume_until	(	const T &	expected,
		u32	end = `END_OF_STREAM`,
		bool	level_aware = `true`,
		bool	throw_on_error = `false`
	)

inline

Consume the next tokens in the stream until a token matches the given expected string.
This final token is not consumed, i.e., it is now the next token in the stream.
Consumes tokens until reaching the given end position, if no token matches the given string.
Can be set to be level-aware with respect to the configured increase and decrease level tokens.

Parameters

[in]	expected	- The string on which to end.
[in]	end	- The absolute position in the stream on which to stop if no match was found until this point.
[in]	level_aware	- If false, tokens are also matched if they are not at level 0.
[in]	throw_on_error	- If true, throws an TokenStreamException instead of returning false on error.

Returns: The last consumed token or the last token of the stream.

Definition at line 266 of file token_stream.h.

◆ consumed()

template<typename T >

u32 hal::TokenStream< T >::consumed ( ) const

inline

Get the total number of consumed elements in the token stream.

Returns: The number of consumed tokens.

Definition at line 484 of file token_stream.h.

◆ extract_until()

template<typename T >

TokenStream<T> hal::TokenStream< T >::extract_until	(	const T &	expected,
		u32	end = `END_OF_STREAM`,
		bool	level_aware = `true`,
		bool	throw_on_error = `false`
	)

inline

Consume the next tokens in the stream until a token matches the given expected string.
This final token is not consumed, i.e., it is now the next token in the stream.
All consumed tokens are returned as a new token stream.
Consumes tokens until reaching the given end position, if no token matches the given string.
Can be set to be level-aware with respect to the configured increase and decrease level tokens.

Parameters

[in]	expected	- The string on which to end.
[in]	end	- The absolute position in the stream on which to stop if no match was not found until this point.
[in]	level_aware	- If false, tokens are also matched if they are not at level 0.
[in]	throw_on_error	- If true, throws an TokenStreamException instead of returning false on error.

Returns: All consumed tokens in form of a new token stream.

Definition at line 308 of file token_stream.h.

◆ find_next()

template<typename T >

u32 hal::TokenStream< T >::find_next	(	const T &	match,
		u32	end = `END_OF_STREAM`,
		bool	level_aware = `true`
	)		const

inline

Get the absolute position in the stream of the next matching token.
If no token is found until the given end position, the end position is returned.
Can be set to be level-aware with respect to the configured increase and decrease level tokens.

Parameters

[in]	match	- The string to match.
[in]	end	- The absolute position in the stream on which to stop if no match was not found until this point.
[in]	level_aware	- If false, tokens are also matched if they are not at level 0.

Returns: The token at the queried position or END_OF_STREAM if not found.

Definition at line 446 of file token_stream.h.

◆ join()

template<typename T >

Token<T> hal::TokenStream< T >::join ( const T & joiner )

inline

Consume all remaining tokens in the stream.
The strings of all consumed tokens are joined with the given joiner string and returned as a new token.
The returned token has the line number of the first consumed Token.

Parameters

[in] joiner - The string used to join consumed tokens.

Returns: The joined token.

Definition at line 367 of file token_stream.h.

References hal::T.

◆ join_until()

template<typename T >

Token<T> hal::TokenStream< T >::join_until	(	const T &	match,
		const T &	joiner,
		u32	end = `END_OF_STREAM`,
		bool	level_aware = `true`,
		bool	throw_on_error = `false`
	)

inline

Consume the next tokens in the stream until a token matches the given expected string.
This final token is not consumed, i.e., it is now the next token in the stream.
The strings of all consumed tokens are joined with the given joiner string and returned as a new Token.
The returned token has the line number of the first consumed token.
Joins tokens until reaching the given end position, if no token matches the given string.
Can be set to be level-aware with respect to the configured increase and decrease level tokens.

Parameters

[in]	match	- The string on which to end.
[in]	joiner	- The string used to join consumed tokens.
[in]	end	- The absolute position in the stream on which to stop if no match was not found until this point.
[in]	level_aware	- If false, tokens are also matched if they are not at level 0.
[in]	throw_on_error	- If true, throws an TokenStreamException instead of returning false on error.

Returns: The joined token.

Definition at line 338 of file token_stream.h.

References hal::T.

◆ operator=()

template<typename T >

TokenStream<T>& hal::TokenStream< T >::operator= ( const TokenStream< T > & other )

inline

Assign a token stream.

Parameters

[in] other - The token stream.

Returns: A reference to the token stream.

Definition at line 198 of file token_stream.h.

◆ peek() [1/2]

template<typename T >

Token<T>& hal::TokenStream< T >::peek ( i32 offset = 0 )

inline

Return a token at a relative position in the stream without consuming it.
Can also access tokens that have already been consumed by using negative offsets.
An offset of 0 referse to the next element of the stream.

Throws a std::out_of_range exception if the position is not contained in the stream.

Parameters

[in] offset - The relative offset from the current position in the stream.

Returns: The token at the queried position.

Definition at line 392 of file token_stream.h.

◆ peek() [2/2]

template<typename T >

const Token<T>& hal::TokenStream< T >::peek ( i32 offset = 0 ) const

inline

Return a token at a relative position in the stream without consuming it.
Can also access tokens that have already been consumed by using negative offsets.
An offset of 0 referse to the next element of the stream.

Throws a std::out_of_range exception if the position is not contained in the stream.

Parameters

[in] offset - The relative offset from the current position in the stream.

Returns: The token at the queried position.

Definition at line 400 of file token_stream.h.

◆ position()

template<typename T >

u32 hal::TokenStream< T >::position ( ) const

inline

Get the current absolute position in the token stream.

Returns: The current position.

Definition at line 504 of file token_stream.h.

◆ remaining()

template<typename T >

u32 hal::TokenStream< T >::remaining ( ) const

inline

Get the total number of remaining elements in the token stream.

Returns: The number of remaining tokens.

Definition at line 494 of file token_stream.h.

◆ set_position()

template<typename T >

void hal::TokenStream< T >::set_position ( u32 p )

inline

Set the current absolute position in the token stream.

Parameters

[in] p - The new position.

Definition at line 514 of file token_stream.h.

◆ size()

template<typename T >

u32 hal::TokenStream< T >::size ( ) const

inline

Get the total number of elements in the token stream regardless of how many tokens have been consumed.

Returns: The size of the token stream.

Definition at line 474 of file token_stream.h.

Member Data Documentation

◆ END_OF_STREAM

template<typename T >

const u32 hal::TokenStream< T >::END_OF_STREAM = 0xFFFFFFFF

static

Constant value that marks the end of the token stream.

Definition at line 147 of file token_stream.h.

The documentation for this class was generated from the following file:

/home/runner/work/hal/hal/include/hal_core/utilities/token_stream.h

Classes

Public Member Functions

Static Public Attributes

Detailed Description

template<typename T> class hal::TokenStream< T >

Constructor & Destructor Documentation

◆ TokenStream() [1/3]

◆ TokenStream() [2/3]

◆ TokenStream() [3/3]

Member Function Documentation

◆ at() [1/2]

◆ at() [2/2]

◆ consume() [1/2]

◆ consume() [2/2]

◆ consume_current_line()

◆ consume_until()

◆ consumed()

◆ extract_until()

◆ find_next()

◆ join()

◆ join_until()

◆ operator=()

◆ peek() [1/2]

◆ peek() [2/2]

◆ position()

◆ remaining()

◆ set_position()

◆ size()

Member Data Documentation

◆ END_OF_STREAM

template<typename T>
class hal::TokenStream< T >