diff --git a/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp b/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp index 0f955803e..72e69af75 100644 --- a/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp +++ b/src/Parser/Parsing/Impl/DefinesStreamProxy.cpp @@ -6,63 +6,76 @@ #include "Parsing/Simple/Expression/SimpleExpressionMatchers.h" #include "Parsing/Simple/SimpleExpressionInterpreter.h" #include "Utils/ClassUtils.h" +#include "Utils/StringUtils.h" #include #include #include +namespace +{ + bool IsStringizeParameterForwardLookup(const std::string& value, unsigned pos) + { + return pos + 1 && (isalpha(value[pos + 1]) || value[pos + 1] == '_'); + } + + bool IsTokenPastingOperatorForwardLookup(const std::string& value, unsigned pos) + { + return pos + 1 < value.size() && value[pos + 1] == '#'; + } +} // namespace + DefinesStreamProxy::DefineParameterPosition::DefineParameterPosition() : m_parameter_index(0u), - m_parameter_position(0u) + m_parameter_position(0u), + m_stringize(false) { } -DefinesStreamProxy::DefineParameterPosition::DefineParameterPosition(const unsigned index, const unsigned position) +DefinesStreamProxy::DefineParameterPosition::DefineParameterPosition(const unsigned index, const unsigned position, const bool stringize) : m_parameter_index(index), - m_parameter_position(position) + m_parameter_position(position), + m_stringize(stringize) { } -DefinesStreamProxy::Define::Define() = default; +DefinesStreamProxy::Define::Define() + : m_contains_token_pasting_operators(false){}; DefinesStreamProxy::Define::Define(std::string name, std::string value) : m_name(std::move(name)), - m_value(std::move(value)) + m_value(std::move(value)), + m_contains_token_pasting_operators(false) { } -std::string DefinesStreamProxy::Define::Render(const std::vector& parameterValues) const +DefinesStreamProxy::MacroParameterState::MacroParameterState() + : m_parameter_state(ParameterState::NOT_IN_PARAMETERS) { - if (parameterValues.empty() || m_parameter_positions.empty()) - return m_value; +} - std::ostringstream str; - auto lastPos = 0u; - for (const auto& parameterPosition : m_parameter_positions) +void DefinesStreamProxy::Define::IdentifyTokenPasteOperatorOnly() +{ + for (auto i = 0u; i < m_value.size(); i++) { - if (lastPos < parameterPosition.m_parameter_position) - str << std::string(m_value, lastPos, parameterPosition.m_parameter_position - lastPos); - - if (parameterPosition.m_parameter_index < parameterValues.size()) + if (m_value[i] == '#' && IsTokenPastingOperatorForwardLookup(m_value, i)) { - str << parameterValues[parameterPosition.m_parameter_index]; + m_contains_token_pasting_operators = true; + return; } - - lastPos = parameterPosition.m_parameter_position; } - - if (lastPos < m_value.size()) - str << std::string(m_value, lastPos, m_value.size() - lastPos); - - return str.str(); } void DefinesStreamProxy::Define::IdentifyParameters(const std::vector& parameterNames) { if (parameterNames.empty()) + { + IdentifyTokenPasteOperatorOnly(); return; + } auto inWord = false; + auto stringizeNext = false; auto wordStart = 0u; for (auto i = 0u; i < m_value.size(); i++) { @@ -82,12 +95,28 @@ void DefinesStreamProxy::Define::IdentifyParameters(const std::vector DefinesStreamProxy::ParseExpression(std::shar { ParserLine pseudoLine(std::move(fileName), lineNumber, std::move(expressionString)); ExpandDefinedExpressions(pseudoLine); - ExpandDefines(pseudoLine); + ProcessMacrosMultiLine(pseudoLine); std::istringstream ss(pseudoLine.m_line); ParserSingleInputStream inputStream(ss, "defines directive expression"); @@ -475,10 +505,13 @@ bool DefinesStreamProxy::MatchDirectives(ParserLine& line) || MatchEndifDirective(line, directiveStartPos, directiveEndPos); } -bool DefinesStreamProxy::FindDefineForWord(const std::string& line, const unsigned wordStart, const unsigned wordEnd, const Define*& value) const +bool DefinesStreamProxy::FindMacroForIdentifier(const std::string& input, + const unsigned identifierStart, + const unsigned identifierEnd, + const Define*& value) const { - const std::string word(line, wordStart, wordEnd - wordStart); - const auto foundEntry = m_defines.find(word); + const std::string identifier(input, identifierStart, identifierEnd - identifierStart); + const auto foundEntry = m_defines.find(identifier); if (foundEntry != m_defines.end()) { value = &foundEntry->second; @@ -488,120 +521,20 @@ bool DefinesStreamProxy::FindDefineForWord(const std::string& line, const unsign return false; } -void DefinesStreamProxy::ContinueMacroParameters(const ParserLine& line, unsigned& pos) +void DefinesStreamProxy::ExtractParametersFromMacroUsage( + const ParserLine& line, unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos) { - const auto lineLength = line.m_line.size(); - while (m_macro_parameter_state != ParameterState::NOT_IN_PARAMETERS && pos < lineLength) - { - const auto c = line.m_line[pos]; - - if (c == ',') - { - if (!m_macro_bracket_depth.empty()) - { - m_macro_parameter_state = ParameterState::AFTER_PARAM; - m_current_macro_parameter << c; - } - else - { - m_macro_parameters.emplace_back(m_current_macro_parameter.str()); - m_current_macro_parameter.clear(); - m_current_macro_parameter.str(std::string()); - m_macro_parameter_state = ParameterState::AFTER_COMMA; - } - } - else if (c == '(' || c == '[' || c == '{') - { - m_macro_parameter_state = ParameterState::AFTER_PARAM; - m_macro_bracket_depth.push(c); - m_current_macro_parameter << c; - } - else if (c == ')') - { - if (!m_macro_bracket_depth.empty()) - { - if (m_macro_bracket_depth.top() != '(') - throw ParsingException(CreatePos(line, pos), "Unbalanced brackets in macro parameters"); - - m_macro_bracket_depth.pop(); - m_macro_parameter_state = ParameterState::AFTER_PARAM; - m_current_macro_parameter << c; - } - else if (m_macro_parameter_state == ParameterState::AFTER_COMMA) - { - throw ParsingException(CreatePos(line, pos), "Cannot close macro parameters after comma"); - } - else - { - m_macro_parameters.emplace_back(m_current_macro_parameter.str()); - m_macro_parameter_state = ParameterState::NOT_IN_PARAMETERS; - } - } - else if (c == ']' || c == '}') - { - if (!m_macro_bracket_depth.empty()) - { - const auto otherBracket = c == ']' ? '[' : '{'; - if (m_macro_bracket_depth.top() != otherBracket) - throw ParsingException(CreatePos(line, pos), "Unbalanced brackets in macro parameters"); - m_macro_bracket_depth.pop(); - } - - m_macro_parameter_state = ParameterState::AFTER_PARAM; - m_current_macro_parameter << c; - } - else if (m_macro_parameter_state == ParameterState::AFTER_PARAM || !isspace(c)) - { - m_macro_parameter_state = ParameterState::AFTER_PARAM; - m_current_macro_parameter << c; - } - - pos++; - } -} - -void DefinesStreamProxy::ContinueMacro(ParserLine& line) -{ - auto pos = 0u; - ContinueMacroParameters(line, pos); - - if (m_macro_parameter_state == ParameterState::NOT_IN_PARAMETERS) - { - const auto defineValue = m_current_macro->Render(m_macro_parameters); - - if (pos < line.m_line.size()) - { - std::ostringstream ss; - ss << defineValue; - ss << std::string(line.m_line, pos, line.m_line.size() - pos); - line.m_line = ss.str(); - } - else - { - line.m_line = defineValue; - } - - ExpandDefines(line); - } - else - { - line.m_line = ""; - } -} - -void DefinesStreamProxy::ExtractParametersFromDefineUsage(const ParserLine& line, const unsigned parameterStart, unsigned& parameterEnd) -{ - if (line.m_line[parameterStart] != '(') + if (input[inputPos] != '(') return; - m_macro_parameter_state = ParameterState::AFTER_OPEN; - m_macro_parameters = std::vector(); - m_current_macro_parameter.clear(); - m_current_macro_parameter.str(std::string()); - m_macro_bracket_depth = std::stack(); - parameterEnd = parameterStart + 1; + inputPos++; + state.m_parameter_state = ParameterState::AFTER_OPEN; + state.m_parameters = std::vector(); + state.m_current_parameter.clear(); + state.m_current_parameter.str(std::string()); + state.m_bracket_depth = std::stack(); - ContinueMacroParameters(line, parameterEnd); + ContinueMacroParameters(line, linePos, state, input, inputPos); } bool DefinesStreamProxy::MatchDefinedExpression(const ParserLine& line, unsigned& pos, std::string& definitionName) @@ -655,53 +588,62 @@ void DefinesStreamProxy::ExpandDefinedExpressions(ParserLine& line) const } } -void DefinesStreamProxy::ProcessDefine(const ParserLine& line, unsigned& pos, std::ostringstream& out) +bool DefinesStreamProxy::FindNextMacro(const std::string& input, unsigned& inputPos, unsigned& defineStart, const DefinesStreamProxy::Define*& define) { - ExtractParametersFromDefineUsage(line, pos, pos); - - if (m_macro_parameter_state == ParameterState::NOT_IN_PARAMETERS) - { - const auto defineValue = m_current_macro->Render(m_macro_parameters); - out << defineValue; - } -} - -bool DefinesStreamProxy::FindNextDefine(const std::string& line, unsigned& pos, unsigned& defineStart, const DefinesStreamProxy::Define*& define) -{ - const auto lineSize = line.size(); + const auto inputSize = input.size(); auto wordStart = 0u; auto lastWordEnd = 0u; auto inWord = false; + auto inString = false; + auto stringEscape = false; - for (; pos < lineSize; pos++) + for (; inputPos < inputSize; inputPos++) { - const auto c = line[pos]; - if (!inWord) + const auto c = input[inputPos]; + if (inString) { - if (isalpha(c) || c == '_') + if (!stringEscape) { - wordStart = pos; - inWord = true; + if (c == '"') + inString = false; + else if (c == '\\') + stringEscape = true; } + else + stringEscape = false; } else { - if (!isalnum(c) && c != '_') + if (c == '"') + inString = true; + + if (!inWord) { - if (FindDefineForWord(line, wordStart, pos, define)) + if (isalpha(c) || c == '_') { - defineStart = wordStart; - return true; + wordStart = inputPos; + inWord = true; } + } + else + { + if (!isalnum(c) && c != '_') + { + if (FindMacroForIdentifier(input, wordStart, inputPos, define)) + { + defineStart = wordStart; + return true; + } - inWord = false; + inWord = false; + } } } } if (inWord) { - if (FindDefineForWord(line, wordStart, pos, define)) + if (FindMacroForIdentifier(input, wordStart, inputPos, define)) { defineStart = wordStart; return true; @@ -711,51 +653,424 @@ bool DefinesStreamProxy::FindNextDefine(const std::string& line, unsigned& pos, return false; } -void DefinesStreamProxy::ExpandDefines(ParserLine& line) +namespace +{ + enum class TokenPasteTokenType + { + NONE, + STRING, + IDENTIFIER, + SYMBOL + }; + + class TokenPasteToken + { + public: + TokenPasteToken() + : m_type(TokenPasteTokenType::NONE), + m_start(0u), + m_end(0u) + { + } + + ~TokenPasteToken() = default; + TokenPasteToken(const TokenPasteToken& other) = default; + TokenPasteToken(TokenPasteToken&& other) = default; + TokenPasteToken& operator=(const TokenPasteToken& other) = default; + TokenPasteToken& operator=(TokenPasteToken&& other) noexcept = default; + + void SetFromInput(ParserLine& line, unsigned& linePos, const std::string& input, unsigned& offset) + { + m_start = offset; + + const auto firstChar = input[offset++]; + const auto inputSize = input.size(); + if (firstChar == '"') + { + m_type = TokenPasteTokenType::STRING; + for (; offset < inputSize; offset++) + { + const auto c = input[offset]; + if (c == '\\') + offset++; // Skip next char + else if (c == '"') + break; + } + + if (offset >= inputSize) + throw new ParsingException(TokenPos(*line.m_filename, line.m_line_number, static_cast(linePos + 1)), + "Token-pasting operator cannot be used on unclosed string"); + + offset++; + } + else if (isalpha(firstChar) || firstChar == '_') + { + m_type = TokenPasteTokenType::IDENTIFIER; + for (; offset < inputSize; offset++) + { + const auto c = input[offset]; + if (!isalnum(c) && c != '_') + break; + } + } + else + { + m_type = TokenPasteTokenType::SYMBOL; + } + + m_end = offset; + } + + void EmitValue(std::ostream& out, const std::string& input) const + { + if (m_end <= m_start) + return; + + if (m_type == TokenPasteTokenType::STRING) + { + if (m_end - m_start > 2) + out << std::string(input, m_start + 1, m_end - m_start - 2); + } + else + { + assert(m_type == TokenPasteTokenType::IDENTIFIER || m_type == TokenPasteTokenType::SYMBOL); + out << std::string(input, m_start, m_end - m_start); + } + } + + TokenPasteTokenType m_type; + unsigned m_start; + unsigned m_end; + }; + + void EmitPastedTokens( + ParserLine& line, unsigned& linePos, std::ostream& out, const std::string& input, const TokenPasteToken& token0, const TokenPasteToken& token1) + { + if ((token0.m_type == TokenPasteTokenType::STRING) != (token1.m_type == TokenPasteTokenType::STRING)) + throw new ParsingException(TokenPos(*line.m_filename, line.m_line_number, static_cast(linePos + 1)), + "String token can only use token-pasting operator on other string token"); + if (token0.m_type == TokenPasteTokenType::STRING) + { + out << '"'; + token0.EmitValue(out, input); + token1.EmitValue(out, input); + out << '"'; + } + else + { + assert(token0.m_type == TokenPasteTokenType::IDENTIFIER || token0.m_type == TokenPasteTokenType::SYMBOL); + + token0.EmitValue(out, input); + token1.EmitValue(out, input); + } + } +} // namespace + +void DefinesStreamProxy::ProcessTokenPastingOperators( + ParserLine& line, unsigned& linePos, std::vector& callstack, std::string& input, unsigned& inputPos) +{ + std::ostringstream ss; + + auto pasteNext = false; + TokenPasteToken previousToken; + TokenPasteToken currentToken; + + const auto inputSize = input.size(); + for (auto i = 0u; i < inputSize;) + { + const auto c = input[i]; + + if (isspace(c)) + { + i++; + continue; + } + + if (c == '#' && IsTokenPastingOperatorForwardLookup(input, i)) + { + if (currentToken.m_type == TokenPasteTokenType::NONE) + throw new ParsingException(CreatePos(line, linePos), "Cannot use token-pasting operator without previous token"); + + if (previousToken.m_end < currentToken.m_start) + ss << std::string(input, previousToken.m_end, currentToken.m_start - previousToken.m_end); + + previousToken = currentToken; + pasteNext = true; + + // Skip second # + i += 2; + } + else + { + currentToken.SetFromInput(line, linePos, input, i); + if (pasteNext) + { + EmitPastedTokens(line, linePos, ss, input, previousToken, currentToken); + previousToken = currentToken; + pasteNext = false; + } + } + } + + if (inputSize > previousToken.m_end) + ss << std::string(input, previousToken.m_end, inputSize - previousToken.m_end); + + if (pasteNext) + throw new ParsingException(CreatePos(line, linePos), "Cannot use token-pasting operator without following token"); + + input = ss.str(); +} + +void DefinesStreamProxy::InsertMacroParameters(std::ostringstream& out, const DefinesStreamProxy::Define* macro, std::vector& parameterValues) { - auto defineIterations = 0u; - bool usesDefines; + if (parameterValues.empty() || macro->m_parameter_positions.empty()) + { + out << macro->m_value; + return; + } - do + auto lastPos = 0u; + for (const auto& parameterPosition : macro->m_parameter_positions) { - if (defineIterations > MAX_DEFINE_ITERATIONS) + if (lastPos < parameterPosition.m_parameter_position) + out << std::string(macro->m_value, lastPos, parameterPosition.m_parameter_position - lastPos); + + if (parameterPosition.m_parameter_index < parameterValues.size()) { - throw ParsingException(CreatePos(line, 1), - "Potential define loop? Exceeded max define iterations of " + std::to_string(MAX_DEFINE_ITERATIONS) + " iterations."); + if (!parameterPosition.m_stringize) + { + out << parameterValues[parameterPosition.m_parameter_index]; + } + else + out << '"' << utils::EscapeStringForQuotationMarks(parameterValues[parameterPosition.m_parameter_index]) << '"'; } - usesDefines = false; - auto pos = 0u; - auto defineStart = 0u; - auto lastDefineEnd = 0u; - std::ostringstream str; + lastPos = parameterPosition.m_parameter_position; + } + + if (lastPos < macro->m_value.size()) + out << std::string(macro->m_value, lastPos, macro->m_value.size() - lastPos); +} + +void DefinesStreamProxy::ExpandMacro(ParserLine& line, + unsigned& linePos, + std::ostringstream& out, + std::vector& callstack, + const DefinesStreamProxy::Define* macro, + std::vector& parameterValues) +{ + std::ostringstream rawOutput; + InsertMacroParameters(rawOutput, macro, parameterValues); + + std::string str = rawOutput.str(); + unsigned nestedPos = 0; + ProcessNestedMacros(line, linePos, callstack, str, nestedPos); - while (FindNextDefine(line.m_line, pos, defineStart, m_current_macro)) + if (macro->m_contains_token_pasting_operators) + ProcessTokenPastingOperators(line, linePos, callstack, str, nestedPos); + + out << str; +} + +void DefinesStreamProxy::ContinueMacroParameters( + const ParserLine& line, unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos) +{ + const auto inputLength = input.size(); + while (state.m_parameter_state != ParameterState::NOT_IN_PARAMETERS && inputPos < inputLength) + { + const auto c = input[inputPos]; + + if (c == ',') { - if (!usesDefines) + if (!state.m_bracket_depth.empty()) { - usesDefines = true; - str << std::string(line.m_line, 0, defineStart); + state.m_parameter_state = ParameterState::AFTER_PARAM; + state.m_current_parameter << c; } else { - str << std::string(line.m_line, lastDefineEnd, defineStart - (lastDefineEnd)); + state.m_parameters.emplace_back(state.m_current_parameter.str()); + state.m_current_parameter.clear(); + state.m_current_parameter.str(std::string()); + state.m_parameter_state = ParameterState::AFTER_COMMA; } + } + else if (c == '(' || c == '[' || c == '{') + { + state.m_parameter_state = ParameterState::AFTER_PARAM; + state.m_bracket_depth.push(c); + state.m_current_parameter << c; + } + else if (c == ')') + { + if (!state.m_bracket_depth.empty()) + { + if (state.m_bracket_depth.top() != '(') + throw ParsingException(CreatePos(line, linePos), "Unbalanced brackets in macro parameters"); - ProcessDefine(line, pos, str); + state.m_bracket_depth.pop(); + state.m_parameter_state = ParameterState::AFTER_PARAM; + state.m_current_parameter << c; + } + else if (state.m_parameter_state == ParameterState::AFTER_COMMA) + { + throw ParsingException(CreatePos(line, linePos), "Cannot close macro parameters after comma"); + } + else + { + state.m_parameters.emplace_back(state.m_current_parameter.str()); + state.m_parameter_state = ParameterState::NOT_IN_PARAMETERS; + } + } + else if (c == ']' || c == '}') + { + if (!state.m_bracket_depth.empty()) + { + const auto otherBracket = c == ']' ? '[' : '{'; + if (state.m_bracket_depth.top() != otherBracket) + throw ParsingException(CreatePos(line, linePos), "Unbalanced brackets in macro parameters"); + state.m_bracket_depth.pop(); + } + + state.m_parameter_state = ParameterState::AFTER_PARAM; + state.m_current_parameter << c; + } + else if (state.m_parameter_state == ParameterState::AFTER_PARAM || !isspace(c)) + { + state.m_parameter_state = ParameterState::AFTER_PARAM; + state.m_current_parameter << c; + } + + inputPos++; + } +} + +void DefinesStreamProxy::ContinueMultiLineMacro(ParserLine& line) +{ + auto pos = 0u; + ContinueMacroParameters(line, pos, m_multi_line_macro_parameters, line.m_line, pos); + + if (m_multi_line_macro_parameters.m_parameter_state == ParameterState::NOT_IN_PARAMETERS) + { + std::ostringstream ss; + std::vector callstack; + ExpandMacro(line, pos, ss, callstack, m_current_macro, m_multi_line_macro_parameters.m_parameters); + + if (pos < line.m_line.size()) + ss << std::string(line.m_line, pos, line.m_line.size() - pos); + + line.m_line = ss.str(); + + ProcessMacrosMultiLine(line); + } + else + { + line.m_line = std::string(); + } +} + +void DefinesStreamProxy::ProcessNestedMacros(ParserLine& line, unsigned& linePos, std::vector& callstack, std::string& input, unsigned& inputPos) +{ + bool usesDefines = false; + + auto pos = 0u; + auto defineStart = 0u; + auto lastDefineEnd = 0u; + std::ostringstream ss; + + const Define* nestedMacro = nullptr; + while (FindNextMacro(input, pos, defineStart, nestedMacro)) + { + if (std::find(callstack.cbegin(), callstack.cend(), nestedMacro) != callstack.cend()) + { + // Do not expand recursively + continue; + } - lastDefineEnd = pos; + // Make sure we account for all text between the last macro (or beginning) and now + if (!usesDefines) + { + usesDefines = true; + ss << std::string(input, 0, defineStart); + } + else + { + ss << std::string(input, lastDefineEnd, defineStart - (lastDefineEnd)); } - if (usesDefines) + callstack.push_back(nestedMacro); + + MacroParameterState nestedMacroState; + ExtractParametersFromMacroUsage(line, linePos, nestedMacroState, input, pos); + if (nestedMacroState.m_parameter_state != ParameterState::NOT_IN_PARAMETERS) + throw ParsingException(CreatePos(line, linePos), "Unbalanced brackets in macro parameters"); + ExpandMacro(line, linePos, ss, callstack, nestedMacro, nestedMacroState.m_parameters); + + callstack.pop_back(); + + lastDefineEnd = pos; + } + + if (usesDefines) + { + // Make sure we account for all text between the last macro and the end + if (lastDefineEnd < input.size()) + ss << std::string(input, lastDefineEnd, input.size() - lastDefineEnd); + input = ss.str(); + } +} + +void DefinesStreamProxy::ProcessMacrosSingleLine(ParserLine& line) +{ + unsigned pos = 0; + std::vector callstack; + ProcessNestedMacros(line, pos, callstack, line.m_line, pos); +} + +void DefinesStreamProxy::ProcessMacrosMultiLine(ParserLine& line) +{ + bool usesDefines = false; + + auto pos = 0u; + auto defineStart = 0u; + auto lastDefineEnd = 0u; + std::ostringstream str; + std::vector callstack; + while (FindNextMacro(line.m_line, pos, defineStart, m_current_macro)) + { + // Make sure we account for all text between the last macro (or beginning) and now + if (!usesDefines) { - if (lastDefineEnd < line.m_line.size()) - str << std::string(line.m_line, lastDefineEnd, line.m_line.size() - lastDefineEnd); - line.m_line = str.str(); + usesDefines = true; + str << std::string(line.m_line, 0, defineStart); } + else + { + str << std::string(line.m_line, lastDefineEnd, defineStart - (lastDefineEnd)); + } + + callstack.push_back(m_current_macro); + + ExtractParametersFromMacroUsage(line, pos, m_multi_line_macro_parameters, line.m_line, pos); - defineIterations++; - } while (usesDefines); + // If still in parameters they continue on the next line + if (m_multi_line_macro_parameters.m_parameter_state == ParameterState::NOT_IN_PARAMETERS) + { + ExpandMacro(line, pos, str, callstack, m_current_macro, m_multi_line_macro_parameters.m_parameters); + } + + callstack.pop_back(); + + lastDefineEnd = pos; + } + + if (usesDefines) + { + // Make sure we account for all text between the last macro and the end + if (lastDefineEnd < line.m_line.size()) + str << std::string(line.m_line, lastDefineEnd, line.m_line.size() - lastDefineEnd); + line.m_line = str.str(); + } } void DefinesStreamProxy::AddDefine(Define define) @@ -796,9 +1111,9 @@ ParserLine DefinesStreamProxy::NextLine() line = m_stream->NextLine(); } - else if (m_macro_parameter_state != ParameterState::NOT_IN_PARAMETERS) + else if (m_multi_line_macro_parameters.m_parameter_state != ParameterState::NOT_IN_PARAMETERS) { - ContinueMacro(line); + ContinueMultiLineMacro(line); return line; } else if (MatchDirectives(line) || !m_modes.empty() && m_modes.top() != BlockMode::IN_BLOCK) @@ -813,7 +1128,7 @@ ParserLine DefinesStreamProxy::NextLine() } else { - ExpandDefines(line); + ProcessMacrosMultiLine(line); return line; } } diff --git a/src/Parser/Parsing/Impl/DefinesStreamProxy.h b/src/Parser/Parsing/Impl/DefinesStreamProxy.h index cedd48c01..21ff4fad5 100644 --- a/src/Parser/Parsing/Impl/DefinesStreamProxy.h +++ b/src/Parser/Parsing/Impl/DefinesStreamProxy.h @@ -5,8 +5,10 @@ #include "Parsing/Simple/Expression/ISimpleExpression.h" #include +#include #include #include +#include class DefinesStreamProxy final : public AbstractDirectiveStreamProxy { @@ -28,9 +30,10 @@ class DefinesStreamProxy final : public AbstractDirectiveStreamProxy public: unsigned m_parameter_index; unsigned m_parameter_position; + bool m_stringize; DefineParameterPosition(); - DefineParameterPosition(unsigned index, unsigned position); + DefineParameterPosition(unsigned index, unsigned position, bool stringize); }; class Define @@ -39,19 +42,14 @@ class DefinesStreamProxy final : public AbstractDirectiveStreamProxy std::string m_name; std::string m_value; std::vector m_parameter_positions; + bool m_contains_token_pasting_operators; Define(); Define(std::string name, std::string value); void IdentifyParameters(const std::vector& parameterNames); - _NODISCARD std::string Render(const std::vector& parameterValues) const; - }; -private: - enum class BlockMode : uint8_t - { - NOT_IN_BLOCK, - IN_BLOCK, - BLOCK_BLOCKED + private: + void IdentifyTokenPasteOperatorOnly(); }; enum class ParameterState : uint8_t @@ -62,6 +60,26 @@ class DefinesStreamProxy final : public AbstractDirectiveStreamProxy AFTER_COMMA }; + class MacroParameterState + { + public: + ParameterState m_parameter_state; + std::ostringstream m_current_parameter; + std::vector m_parameters; + std::stack m_bracket_depth; + + MacroParameterState(); + }; + +private: + enum class BlockMode : uint8_t + + { + NOT_IN_BLOCK, + IN_BLOCK, + BLOCK_BLOCKED + }; + IParserLineStream* const m_stream; const bool m_skip_directive_lines; std::map m_defines; @@ -75,10 +93,7 @@ class DefinesStreamProxy final : public AbstractDirectiveStreamProxy std::vector m_current_define_parameters; const Define* m_current_macro; - ParameterState m_macro_parameter_state; - std::vector m_macro_parameters; - std::ostringstream m_current_macro_parameter; - std::stack m_macro_bracket_depth; + MacroParameterState m_multi_line_macro_parameters; static int GetLineEndEscapePos(const ParserLine& line); void MatchDefineParameters(const ParserLine& line, unsigned& currentPos); @@ -93,16 +108,29 @@ class DefinesStreamProxy final : public AbstractDirectiveStreamProxy _NODISCARD bool MatchEndifDirective(const ParserLine& line, unsigned directiveStartPosition, unsigned directiveEndPosition); _NODISCARD bool MatchDirectives(ParserLine& line); - void ExtractParametersFromDefineUsage(const ParserLine& line, unsigned parameterStart, unsigned& parameterEnd); - bool FindDefineForWord(const std::string& line, unsigned wordStart, unsigned wordEnd, const Define*& value) const; + void ExtractParametersFromMacroUsage(const ParserLine& line, unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos); + bool FindMacroForIdentifier(const std::string& input, unsigned wordStart, unsigned wordEnd, const Define*& value) const; static bool MatchDefinedExpression(const ParserLine& line, unsigned& pos, std::string& definitionName); void ExpandDefinedExpressions(ParserLine& line) const; - void ContinueMacroParameters(const ParserLine& line, unsigned& pos); - void ContinueMacro(ParserLine& line); - void ProcessDefine(const ParserLine& line, unsigned& pos, std::ostringstream& out); - bool FindNextDefine(const std::string& line, unsigned& pos, unsigned& defineStart, const DefinesStreamProxy::Define*& define); + bool FindNextMacro(const std::string& input, unsigned& inputPos, unsigned& defineStart, const DefinesStreamProxy::Define*& define); + + void ProcessTokenPastingOperators(ParserLine& line, unsigned& linePos, std::vector& callstack, std::string& input, unsigned& inputPos); + void InsertMacroParameters(std::ostringstream& out, const DefinesStreamProxy::Define* macro, std::vector& parameterValues); + void ExpandMacro(ParserLine& line, + unsigned& linePos, + std::ostringstream& out, + std::vector& callstack, + const DefinesStreamProxy::Define* macro, + std::vector& parameterValues); + + void ContinueMacroParameters(const ParserLine& line, unsigned& linePos, MacroParameterState& state, const std::string& input, unsigned& inputPos); + void ContinueMultiLineMacro(ParserLine& line); + + void ProcessNestedMacros(ParserLine& line, unsigned& linePos, std::vector& callstack, std::string& input, unsigned& inputPos); + void ProcessMacrosSingleLine(ParserLine& line); + void ProcessMacrosMultiLine(ParserLine& line); public: explicit DefinesStreamProxy(IParserLineStream* stream, bool skipDirectiveLines = false); @@ -110,8 +138,6 @@ class DefinesStreamProxy final : public AbstractDirectiveStreamProxy void AddDefine(Define define); void Undefine(const std::string& name); - void ExpandDefines(ParserLine& line); - _NODISCARD std::unique_ptr ParseExpression(std::shared_ptr fileName, int lineNumber, std::string expressionString); ParserLine NextLine() override; diff --git a/test/ParserTests/Parsing/Impl/DefinesStreamProxyTests.cpp b/test/ParserTests/Parsing/Impl/DefinesStreamProxyTests.cpp index 6ca696363..a41e8b845 100644 --- a/test/ParserTests/Parsing/Impl/DefinesStreamProxyTests.cpp +++ b/test/ParserTests/Parsing/Impl/DefinesStreamProxyTests.cpp @@ -357,27 +357,21 @@ namespace test::parsing::impl::defines_stream_proxy TEST_CASE("DefinesStreamProxy: Ensure define can render parameters", "[parsing][parsingstream]") { - DefinesStreamProxy::Define define("helloworld", "hello universe"); - - std::vector parameterNames({"universe"}); - define.IdentifyParameters(parameterNames); - - std::vector parameterValues({"mr moneyman"}); - REQUIRE(define.Render(parameterValues) == "hello mr moneyman"); - } + const std::vector lines{ + "#define test(universe) hello universe", + "test(mr moneyman)", + }; - TEST_CASE("DefinesStreamProxy: Ensure define can render parameters in middle of symbols", "[parsing][parsingstream]") - { - DefinesStreamProxy::Define define("helloworld", "alignas(x)"); + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); - std::vector parameterNames({"x"}); - define.IdentifyParameters(parameterNames); + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "hello mr moneyman"); - std::vector parameterValues({"1337"}); - REQUIRE(define.Render(parameterValues) == "alignas(1337)"); + REQUIRE(proxy.Eof()); } - TEST_CASE("DefinesStreamProxy: Ensure can add define with parameters", "[parsing][parsingstream]") + TEST_CASE("DefinesStreamProxy: Ensure can add define with parameters surrounded by symbols", "[parsing][parsingstream]") { const std::vector lines{ "#define test(x) alignas(x)", @@ -473,6 +467,22 @@ namespace test::parsing::impl::defines_stream_proxy REQUIRE(proxy.Eof()); } + TEST_CASE("DefinesStreamProxy: Ensure does not expand macros in strings", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define TEST Wrong", + "System.out.println(\"TEST\")", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "System.out.println(\"TEST\")"); + + REQUIRE(proxy.Eof()); + } + TEST_CASE("DefinesStreamProxy: Ensure simple if is working with truthy value", "[parsing][parsingstream]") { const std::vector lines{ @@ -900,4 +910,320 @@ namespace test::parsing::impl::defines_stream_proxy REQUIRE(proxy.Eof()); } + + TEST_CASE("DefinesStreamProxy: Can use strinizing operator", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define testMacro(a) #a", + "testMacro(Hello)", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "\"Hello\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can use stringizing operator inside sample code", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define testMacro(a) System.out.println(#a)", + "testMacro(Hello)", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "System.out.println(\"Hello\")"); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Stringization does not expand macros", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define TEST WRONG", + "#define TESTTWO(b) WRONG WITH ARG b", + "#define STR(a) #a", + "STR(TEST)", + "STR(TESTTWO(testArg))", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, ""); + ExpectLine(&proxy, 3, ""); + ExpectLine(&proxy, 4, "\"TEST\""); + ExpectLine(&proxy, 5, "\"TESTTWO(testArg)\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can use token-pasting operator with identifier", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define testMacro(a) Hello##a", + "testMacro(World)", + "testMacro(5)", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "HelloWorld"); + ExpectLine(&proxy, 3, "Hello5"); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can use token-pasting operator with string", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define testMacro(a) \"Hello\"##a", + "testMacro(\"World\")", + "testMacro(\"5\")", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "\"HelloWorld\""); + ExpectLine(&proxy, 3, "\"Hello5\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can token-join symbols", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define GLUE(a, b) a ## b", + "GLUE(+, =)", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "+="); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can token-join symbols and identifiers", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define GLUE(a, b) a ## b", + "GLUE(+, hello)", + "GLUE(world, =)", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "+hello"); + ExpectLine(&proxy, 3, "world="); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can token-join strings", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define GLUE(a, b) a ## b", + "GLUE(\"Hello\", \"World\")", + "GLUE(\"\", \"Cat\")", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "\"HelloWorld\""); + ExpectLine(&proxy, 3, "\"Cat\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Combined string tokens keep escape sequences", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define GLUE(a, b) a ## b", + "GLUE(\"He\\\"llo\", \"W\\\\orld\")", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "\"He\\\"lloW\\\\orld\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can use token-pasting operator with string and stringization outside macro", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define s(t) #t", + "#define testMacro(a) \"Hello\" ## a", + "testMacro(s(World))", + "testMacro(s(5))", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, ""); + ExpectLine(&proxy, 3, "\"HelloWorld\""); + ExpectLine(&proxy, 4, "\"Hello5\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can use token-pasting operator with string and stringization inside macro", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define s(t) #t", + "#define testMacro(a) \"Hello\" ## s(a)", + "testMacro(World)", + "testMacro(5)", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, ""); + ExpectLine(&proxy, 3, "\"HelloWorld\""); + ExpectLine(&proxy, 4, "\"Hello5\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can omit whitespace when using token-pasting operator in combination with stringization macro", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define s(t) #t", + "#define testMacro(a) \"Hello\"##s(a)", + "testMacro(World)", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, ""); + ExpectLine(&proxy, 3, "\"HelloWorld\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can omit whitespace when using token-pasting operator and stringization on same macro parameter", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define testMacro(a) \"Hello\"###a", + "testMacro(World)", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "\"HelloWorld\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can use token-pasting operator after length deflating macros", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define VERY_LONG_MACRO \"T\"", + "#define testMacro(a) VERY_LONG_MACRO VERY_LONG_MACRO \"Hello\"##a VERY_LONG_MACRO", + "testMacro(\"World\")", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, ""); + ExpectLine(&proxy, 3, "\"T\" \"T\" \"HelloWorld\" \"T\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Can use token-pasting operator after length inflating macros", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define BLA \"This is very long\"", + "#define testMacro(a) BLA BLA \"Hello\"##a BLA", + "testMacro(\"World\")", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, ""); + ExpectLine(&proxy, 3, "\"This is very long\" \"This is very long\" \"HelloWorld\" \"This is very long\""); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Interprets nested macros in context of top-level macro", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define TEST HELLO", + "TEST", + "#define HELLO ONE", + "TEST", + "#undef HELLO", + "TEST", + "#define HELLO TWO", + "TEST", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, "HELLO"); + ExpectLine(&proxy, 3, ""); + ExpectLine(&proxy, 4, "ONE"); + ExpectLine(&proxy, 5, ""); + ExpectLine(&proxy, 6, "HELLO"); + ExpectLine(&proxy, 7, ""); + ExpectLine(&proxy, 8, "TWO"); + + REQUIRE(proxy.Eof()); + } + + TEST_CASE("DefinesStreamProxy: Recursive macro usages stops upon first reuse of same macro", "[parsing][parsingstream]") + { + const std::vector lines{ + "#define ONE TWO", + "#define TWO THREE", + "#define THREE ONE", + "ONE", + }; + + MockParserLineStream mockStream(lines); + DefinesStreamProxy proxy(&mockStream); + + ExpectLine(&proxy, 1, ""); + ExpectLine(&proxy, 2, ""); + ExpectLine(&proxy, 3, ""); + ExpectLine(&proxy, 4, "ONE"); + + REQUIRE(proxy.Eof()); + } } // namespace test::parsing::impl::defines_stream_proxy