Skip to content

Improve attribute parsing #82

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xoofx merged 2 commits into from
Oct 14, 2023
Merged

Improve attribute parsing #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bfb91f0
Improve attribute parsing
sombalazs Oct 2, 2023
93ac9e7
Merge branch 'main' into improve_attribute_parsing
xoofx Oct 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions src/CppAst.Tests/AttributesTest/TestTokenAttributes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,34 @@ struct [[deprecated(""old"")]] TestMessage{
);
}

[Test]
public void TestCpp11StructAttributesWithMacro()
{
ParseAssert(@"
#define CLASS_ATTRIBUTE [[complex_attribute::attribute_name(""attribute_argument"")]]
struct
CLASS_ATTRIBUTE
Test{
int a;
int b;
};", compilation =>
{
Assert.False(compilation.HasErrors);

Assert.AreEqual(1, compilation.Classes.Count);
Assert.AreEqual(1, compilation.Classes[0].TokenAttributes.Count);
{
var attr = compilation.Classes[0].TokenAttributes[0];
Assert.AreEqual("complex_attribute", attr.Scope);
Assert.AreEqual("attribute_name", attr.Name);
Assert.AreEqual("\"attribute_argument\"", attr.Arguments);
}
},
// we are using a C++14 attribute because it can be used everywhere
new CppParserOptions() { AdditionalArguments = { "-std=c++14" }, ParseTokenAttributes = true }
);
}

[Test]
public void TestCpp11VariablesAttributes()
{
Expand Down Expand Up @@ -204,6 +232,27 @@ public void TestCpp11FunctionsAttributes()
);
}

[Test]
public void TestCpp11FunctionsAttributesOnNewLine()
{
ParseAssert(@"
[[noreturn]]
void x() {};", compilation =>
{
Assert.False(compilation.HasErrors);

Assert.AreEqual(1, compilation.Functions.Count);
Assert.AreEqual(1, compilation.Functions[0].TokenAttributes.Count);
{
var attr = compilation.Functions[0].TokenAttributes[0];
Assert.AreEqual("noreturn", attr.Name);
}
},
// we are using a C++14 attribute because it can be used everywhere
new CppParserOptions() { AdditionalArguments = { "-std=c++14" }, ParseTokenAttributes = true }
);
}

[Test]
public void TestCpp11NamespaceAttributes()
{
Expand Down
6 changes: 3 additions & 3 deletions src/CppAst/CppModelBuilder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1541,19 +1541,19 @@ private void ParseAttributes(CXCursor cursor, ICppAttributeContainer attrContain
bool hasOnlineAttribute = CppTokenUtil.TryToSeekOnlineAttributes(cursor, out var onLineRange);
if (hasOnlineAttribute)
{
CppTokenUtil.ParseAttributesInRange(cursor.TranslationUnit, onLineRange, ref tokenAttributes);
CppTokenUtil.ParseAttributesInRange(_rootContainerContext.Container as CppGlobalDeclarationContainer, cursor.TranslationUnit, onLineRange, ref tokenAttributes);
}
}

//Parse attributes contains in cursor
if (attrContainer is CppFunction)
{
var func = attrContainer as CppFunction;
CppTokenUtil.ParseFunctionAttributes(cursor, func.Name, ref tokenAttributes);
CppTokenUtil.ParseFunctionAttributes(_rootContainerContext.Container as CppGlobalDeclarationContainer, cursor, func.Name, ref tokenAttributes);
}
else
{
CppTokenUtil.ParseCursorAttributs(cursor, ref tokenAttributes);
CppTokenUtil.ParseCursorAttributs(_rootContainerContext.Container as CppGlobalDeclarationContainer, cursor, ref tokenAttributes);
}

attrContainer.TokenAttributes.AddRange(tokenAttributes);
Expand Down
88 changes: 74 additions & 14 deletions src/CppAst/CppTokenUtil.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ namespace CppAst
{
static internal unsafe class CppTokenUtil
{
public static void ParseCursorAttributs(CXCursor cursor, ref List<CppAttribute> attributes)
public static void ParseCursorAttributs(CppGlobalDeclarationContainer globalContainer, CXCursor cursor, ref List<CppAttribute> attributes)
{
var tokenizer = new AttributeTokenizer(cursor);
var tokenIt = new TokenIterator(tokenizer);
Expand All @@ -25,7 +25,7 @@ public static void ParseCursorAttributs(CXCursor cursor, ref List<CppAttribute>

while (tokenIt.CanPeek)
{
if (ParseAttributes(tokenIt, ref attributes))
if (ParseAttributes(globalContainer, tokenIt, ref attributes))
{
continue;
}
Expand All @@ -42,7 +42,7 @@ public static void ParseCursorAttributs(CXCursor cursor, ref List<CppAttribute>
}


public static void ParseFunctionAttributes(CXCursor cursor, string functionName, ref List<CppAttribute> attributes)
public static void ParseFunctionAttributes(CppGlobalDeclarationContainer globalContainer, CXCursor cursor, string functionName, ref List<CppAttribute> attributes)
{
// TODO: This function is not 100% correct when parsing tokens up to the function name
// we assume to find the function name immediately followed by a `(`
Expand All @@ -58,7 +58,7 @@ public static void ParseFunctionAttributes(CXCursor cursor, string functionName,
// Parse leading attributes
while (tokenIt.CanPeek)
{
if (ParseAttributes(tokenIt, ref attributes))
if (ParseAttributes(globalContainer, tokenIt, ref attributes))
{
continue;
}
Expand Down Expand Up @@ -103,7 +103,7 @@ public static void ParseFunctionAttributes(CXCursor cursor, string functionName,

while (tokenIt.CanPeek)
{
if (ParseAttributes(tokenIt, ref attributes))
if (ParseAttributes(globalContainer, tokenIt, ref attributes))
{
continue;
}
Expand All @@ -115,7 +115,7 @@ public static void ParseFunctionAttributes(CXCursor cursor, string functionName,
}


public static void ParseAttributesInRange(CXTranslationUnit tu, CXSourceRange range, ref List<CppAttribute> collectAttributes)
public static void ParseAttributesInRange(CppGlobalDeclarationContainer globalContainer, CXTranslationUnit tu, CXSourceRange range, ref List<CppAttribute> collectAttributes)
{
var tokenizer = new AttributeTokenizer(tu, range);
var tokenIt = new TokenIterator(tokenizer);
Expand All @@ -134,7 +134,7 @@ public static void ParseAttributesInRange(CXTranslationUnit tu, CXSourceRange ra

while (tokenIt.CanPeek)
{
if (ParseAttributes(tokenIt, ref collectAttributes))
if (ParseAttributes(globalContainer, tokenIt, ref collectAttributes))
{
continue;
}
Expand Down Expand Up @@ -615,17 +615,17 @@ bool HasInlineTypeDefinition(CXCursor varDecl)
CXSourceLocation GetNextLocation(CXSourceLocation loc, int inc = 1)
{
CXSourceLocation value;
loc.GetSpellingLocation(out var f, out var u, out var z, out var originalOffset);
var offset = IncOffset(inc, z);
var shouldUseLine = (z != 0 && (offset != 0 || offset != uint.MaxValue));
loc.GetSpellingLocation(out var file, out var line, out var column, out var originalOffset);
var signedOffset = (int)column + inc;
var shouldUseLine = (column != 0 && signedOffset > 0);
if (shouldUseLine)
{
value = tu.GetLocation(f, u, offset);
value = tu.GetLocation(file, line, (uint)signedOffset);
}
else
{
offset = IncOffset(inc, originalOffset);
value = tu.GetLocationForOffset(f, offset);
var offset = IncOffset(inc, originalOffset);
value = tu.GetLocationForOffset(file, offset);
}

return value;
Expand Down Expand Up @@ -911,8 +911,42 @@ private enum AttributeLexerParseStatus
Error,
}

private static (string, string) GetNameSpaceAndAttribute(string fullAttribute)
{
string[] colons = { "::" };
string[] tokens = fullAttribute.Split(colons, System.StringSplitOptions.None);
if (tokens.Length == 2)
{
return (tokens[0], tokens[1]);
}
else
{
return (null, tokens[0]);
}
}


private static (string, string) GetNameAndArguments(string name)
{
if (name.Contains("("))
{
Char[] seperator = { '(' };
var argumentTokens = name.Split(seperator, 2);
var length = argumentTokens[1].LastIndexOf(')');
string argument = null;
if (length > 0)
{
argument = argumentTokens[1].Substring(0, length);
}
return (argumentTokens[0], argument);
}
else
{
return (name, null);
}
}

private static bool ParseAttributes(TokenIterator tokenIt, ref List<CppAttribute> attributes)
private static bool ParseAttributes(CppGlobalDeclarationContainer globalContainer, TokenIterator tokenIt, ref List<CppAttribute> attributes)
{
// Parse C++ attributes
// [[<attribute>]]
Expand Down Expand Up @@ -985,6 +1019,32 @@ private static bool ParseAttributes(TokenIterator tokenIt, ref List<CppAttribute
return tokenIt.Skip(")"); ;
}

// See if we have a macro
var value = tokenIt.PeekText();
var macro = globalContainer.Macros.Find(v => v.Name == value);
if (macro != null)
{
if (macro.Value.StartsWith("[[") && macro.Value.EndsWith("]]"))
{
CppAttribute attribute = null;
var fullAttribute = macro.Value.Substring(2, macro.Value.Length - 4);
var (scope, name) = GetNameSpaceAndAttribute(fullAttribute);
var (attributeName, arguments) = GetNameAndArguments(name);

attribute = new CppAttribute(attributeName, AttributeKind.TokenAttribute);
attribute.Scope = scope;
attribute.Arguments = arguments;

if (attributes == null)
{
attributes = new List<CppAttribute>();
}
attributes.Add(attribute);
tokenIt.Next();
return true;
}
}

return false;
}

Expand Down