Skip to content

Commit aedcfb2

Browse files
joke1196sonartech
joke1196
authored and
sonartech
committed
SONARPY-2306 Support annotated return type information conversion from PythonType to FunctionDescriptor (#700)
GitOrigin-RevId: 871ca32467ff8132738289367f0e14d11352d6e8
1 parent 20f6c9f commit aedcfb2

File tree

10 files changed

+366
-22
lines changed

10 files changed

+366
-22
lines changed

‎python-frontend/src/main/java/org/sonar/plugins/python/api/types/v2/SelfType.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,10 @@ public static PythonType of(@Nullable PythonType type) {
105105
return PythonType.UNKNOWN;
106106
}
107107

108+
public static PythonType fromTypeWrapper(TypeWrapper typeWrapper){
109+
return new SelfType(typeWrapper);
110+
}
111+
108112
public PythonType innerType() {
109113
var type = typeWrapper.type();
110114
if (type instanceof ClassType) {

‎python-frontend/src/main/java/org/sonar/python/index/DescriptorsToProtobuf.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,10 @@ private static DescriptorsProtos.FunctionDescriptor toProtobuf(FunctionDescripto
139139
if (definitionLocation != null) {
140140
builder.setDefinitionLocation(toProtobuf(definitionLocation));
141141
}
142+
TypeAnnotationDescriptor returnTypeAnnotationDescriptor = functionDescriptor.typeAnnotationDescriptor();
143+
if (returnTypeAnnotationDescriptor != null) {
144+
builder.setReturnTypeAnnotationDescriptor(toProtobuf(returnTypeAnnotationDescriptor));
145+
}
142146
return builder.build();
143147
}
144148

@@ -255,6 +259,9 @@ private static FunctionDescriptor fromProtobuf(DescriptorsProtos.FunctionDescrip
255259
functionDescriptorProto.getParametersList().forEach(proto -> parameters.add(fromProtobuf(proto)));
256260
LocationInFile definitionLocation = functionDescriptorProto.hasDefinitionLocation() ? fromProtobuf(functionDescriptorProto.getDefinitionLocation()) : null;
257261
String annotatedReturnTypeName = functionDescriptorProto.hasAnnotatedReturnType() ? functionDescriptorProto.getAnnotatedReturnType() : null;
262+
TypeAnnotationDescriptor returnTypeAnnotationDescriptor = functionDescriptorProto.hasReturnTypeAnnotationDescriptor()
263+
? fromProtobuf(functionDescriptorProto.getReturnTypeAnnotationDescriptor())
264+
: null;
258265
return new FunctionDescriptor(
259266
functionDescriptorProto.getName(),
260267
fullyQualifiedName,
@@ -265,8 +272,7 @@ private static FunctionDescriptor fromProtobuf(DescriptorsProtos.FunctionDescrip
265272
functionDescriptorProto.getHasDecorators(),
266273
definitionLocation,
267274
annotatedReturnTypeName,
268-
// TypeAnnotationDescriptor is not serialized in protobuf
269-
null);
275+
returnTypeAnnotationDescriptor);
270276
}
271277

272278
private static FunctionDescriptor.Parameter fromProtobuf(DescriptorsProtos.ParameterDescriptor parameterDescriptorProto) {
@@ -323,4 +329,4 @@ private static TypeAnnotationDescriptor fromProtobuf(SymbolsProtos.Type typeProt
323329
fullyQualifiedName,
324330
typeProto.getIsSelf());
325331
}
326-
}
332+
}

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/PythonTypeToDescriptorConverter.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,12 @@ private static Descriptor convert(FunctionType type) {
109109
.filter(Objects::nonNull)
110110
.toList();
111111

112+
var returnType = type.returnType();
113+
var unwrappedReturnType = returnType.unwrappedType();
114+
var isSelfReturnType = containsSelfType(returnType);
115+
var annotatedReturnTypeName = FullyQualifiedNameHelper.getFullyQualifiedName(unwrappedReturnType).orElse(null);
116+
var returnTypeAnnotationDescriptor = createTypeAnnotationDescriptor(unwrappedReturnType, isSelfReturnType);
117+
112118
// Using FunctionType#name and FunctionType#fullyQualifiedName instead of symbol is only accurate if the function has not been reassigned
113119
// This logic should be revisited when tackling SONARPY-2285
114120
return new FunctionDescriptor(type.name(), type.fullyQualifiedName(),
@@ -118,8 +124,12 @@ private static Descriptor convert(FunctionType type) {
118124
decorators,
119125
type.hasDecorators(),
120126
type.definitionLocation().orElse(null),
121-
null,
122-
null);
127+
annotatedReturnTypeName,
128+
returnTypeAnnotationDescriptor);
129+
}
130+
131+
private static boolean containsSelfType(PythonType returnType){
132+
return returnType instanceof SelfType || returnType.unwrappedType() instanceof SelfType;
123133
}
124134

125135
@VisibleForTesting
@@ -188,7 +198,7 @@ private static Descriptor convert(String parentFqn, String symbolName, UnknownTy
188198

189199
private static FunctionDescriptor.Parameter convert(ParameterV2 parameter) {
190200
var type = parameter.declaredType().type();
191-
var isSelf = type instanceof SelfType;
201+
var isSelf = containsSelfType(type);
192202
var unwrappedType = type.unwrappedType();
193203
var annotatedType = FullyQualifiedNameHelper.getFullyQualifiedName(unwrappedType).orElse(null);
194204
var typeAnnotationDescriptor = createTypeAnnotationDescriptor(unwrappedType, isSelf);
@@ -214,7 +224,9 @@ private static boolean usagesContainAssignment(List<UsageV2> symbolUsages) {
214224

215225
@CheckForNull
216226
private static TypeAnnotationDescriptor createTypeAnnotationDescriptor(PythonType type, boolean isSelf) {
217-
if (type instanceof ClassType classType) {
227+
if (type instanceof SelfType selfType) {
228+
return createTypeAnnotationDescriptor(selfType.innerType(), isSelf);
229+
}else if (type instanceof ClassType classType) {
218230
return new TypeAnnotationDescriptor(classType.name(), TypeAnnotationDescriptor.TypeKind.INSTANCE, List.of(), classType.fullyQualifiedName(), isSelf);
219231
} else if (type instanceof FunctionType functionType) {
220232
return new TypeAnnotationDescriptor(functionType.name(), TypeAnnotationDescriptor.TypeKind.CALLABLE, List.of(), functionType.fullyQualifiedName(), false);

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/converter/TypeAnnotationToPythonTypeConverter.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.util.stream.Collectors;
2323
import org.sonar.plugins.python.api.types.v2.PythonType;
2424
import org.sonar.plugins.python.api.types.v2.SelfType;
25+
import org.sonar.plugins.python.api.types.v2.TypeWrapper;
2526
import org.sonar.python.index.TypeAnnotationDescriptor;
2627
import org.sonar.python.types.v2.LazyUnionType;
2728

@@ -34,14 +35,18 @@ public PythonType convert(ConversionContext context, TypeAnnotationDescriptor ty
3435
if (fullyQualifiedName == null) {
3536
return PythonType.UNKNOWN;
3637
}
38+
if (Boolean.TRUE.equals(type.isSelf())) {
39+
40+
return SelfType.fromTypeWrapper(TypeWrapper.of(typeFromFQN(fullyQualifiedName, context)));
41+
}
3742
// _SpecialForm is the type used for some special types, like Callable, Union, TypeVar, ...
3843
// It comes from CPython impl: https://github.com/python/cpython/blob/e39ae6bef2c357a88e232dcab2e4b4c0f367544b/Lib/typing.py#L439
3944
// This doesn't seem to be very precisely specified in typeshed, because it has special semantic.
4045
// To avoid FPs, we treat it as ANY
4146
if ("typing._SpecialForm".equals(fullyQualifiedName)) {
4247
return PythonType.UNKNOWN;
4348
}
44-
return fullyQualifiedName.isEmpty() ? PythonType.UNKNOWN : context.lazyTypesContext().getOrCreateLazyType(fullyQualifiedName);
49+
return typeFromFQN(fullyQualifiedName, context);
4550
case TYPE:
4651
return context.lazyTypesContext().getOrCreateLazyType("type");
4752
case TYPE_ALIAS:
@@ -59,8 +64,8 @@ public PythonType convert(ConversionContext context, TypeAnnotationDescriptor ty
5964
// SONARPY-2179: This case only makes sense for parameter types, which are not supported yet
6065
return context.lazyTypesContext().getOrCreateLazyType("dict");
6166
case TYPE_VAR:
62-
if(Boolean.TRUE.equals(type.isSelf())){
63-
if(type.args().size() != 1){
67+
if (Boolean.TRUE.equals(type.isSelf())) {
68+
if (type.args().size() != 1) {
6469
return PythonType.UNKNOWN;
6570
}
6671
PythonType innerType = convert(context, type.args().get(0));
@@ -77,6 +82,10 @@ public PythonType convert(ConversionContext context, TypeAnnotationDescriptor ty
7782
}
7883
}
7984

85+
private static PythonType typeFromFQN(String fullyQualifiedName, ConversionContext context) {
86+
return fullyQualifiedName.isEmpty() ? PythonType.UNKNOWN : context.lazyTypesContext().getOrCreateLazyType(fullyQualifiedName);
87+
}
88+
8089
private static final Set<String> EXCLUDING_TYPE_VAR_FQN_PATTERNS = Set.of(
8190
"object",
8291
"^builtins\\.object$",

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TrivialTypeInferenceVisitor.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,8 +118,8 @@ public class TrivialTypeInferenceVisitor extends BaseTreeVisitor {
118118

119119
private static final TypeInferenceMatcher IS_TYPING_SELF = TypeInferenceMatcher.of(
120120
TypeInferenceMatchers.any(
121-
TypeInferenceMatchers.isType("typing.Self"),
122-
TypeInferenceMatchers.isType("typing_extensions.Self")));
121+
TypeInferenceMatchers.withFQN("typing.Self"),
122+
TypeInferenceMatchers.withFQN("typing_extensions.Self")));
123123

124124
private final TypeTable projectLevelTypeTable;
125125
private final String fileId;

‎python-frontend/src/main/java/org/sonar/python/semantic/v2/types/TypeInferenceMatchers.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.sonar.plugins.python.api.types.v2.PythonType;
2121
import org.sonar.python.api.types.v2.matchers.TypeMatchers;
2222
import org.sonar.python.types.v2.matchers.AnyTypePredicate;
23+
import org.sonar.python.types.v2.matchers.HasFQNPredicate;
2324
import org.sonar.python.types.v2.matchers.IsObjectSatisfyingPredicate;
2425
import org.sonar.python.types.v2.matchers.IsSelfTypePredicate;
2526
import org.sonar.python.types.v2.matchers.IsTypePredicate;
@@ -56,4 +57,8 @@ public static TypePredicate isObjectOfType(String fqn) {
5657
public static TypePredicate isSelf() {
5758
return new IsSelfTypePredicate();
5859
}
60+
61+
public static TypePredicate withFQN(String fqn) {
62+
return new HasFQNPredicate(fqn);
63+
}
5964
}

‎python-frontend/src/main/protobuf/descriptors.proto‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,11 @@ message FunctionDescriptor {
6969
bool hasDecorators = 7;
7070
LocationInFile definitionLocation = 8;
7171
optional string annotatedReturnType = 9;
72+
optional Type returnTypeAnnotationDescriptor = 10;
7273
}
7374

7475
message VarDescriptor {
7576
string name = 1;
7677
optional string fully_qualified_name = 2;
7778
optional string annotatedType = 3;
78-
}
79+
}

‎python-frontend/src/test/java/org/sonar/python/index/FunctionDescriptorTest.java‎

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,89 @@ void protobufSerializationWithoutLocationAndWithAnnotatedReturnType() {
169169
assertDescriptorToProtobuf(functionDescriptor);
170170
}
171171

172+
@Test
173+
void protobufSerializationWithReturnTypeAnnotationDescriptor() {
174+
TypeAnnotationDescriptor returnTypeDescriptor = new TypeAnnotationDescriptor(
175+
"str",
176+
TypeAnnotationDescriptor.TypeKind.INSTANCE,
177+
List.of(),
178+
"builtins.str",
179+
false);
180+
List<FunctionDescriptor.Parameter> parameters = new ArrayList<>();
181+
FunctionDescriptor functionDescriptor = new FunctionDescriptor(
182+
"foo",
183+
"mod.foo",
184+
parameters,
185+
false,
186+
false,
187+
Collections.emptyList(),
188+
false,
189+
null,
190+
"str",
191+
returnTypeDescriptor);
192+
assertDescriptorToProtobuf(functionDescriptor);
193+
}
194+
195+
@Test
196+
void protobufSerializationWithReturnTypeAnnotationDescriptorWithComplexType() {
197+
// Test with a union type: int | str
198+
TypeAnnotationDescriptor intType = new TypeAnnotationDescriptor(
199+
"int",
200+
TypeAnnotationDescriptor.TypeKind.INSTANCE,
201+
List.of(),
202+
"builtins.int",
203+
false);
204+
TypeAnnotationDescriptor strType = new TypeAnnotationDescriptor(
205+
"str",
206+
TypeAnnotationDescriptor.TypeKind.INSTANCE,
207+
List.of(),
208+
"builtins.str",
209+
false);
210+
TypeAnnotationDescriptor returnTypeDescriptor = new TypeAnnotationDescriptor(
211+
"int | str",
212+
TypeAnnotationDescriptor.TypeKind.UNION,
213+
List.of(intType, strType),
214+
null,
215+
false);
216+
List<FunctionDescriptor.Parameter> parameters = new ArrayList<>();
217+
FunctionDescriptor functionDescriptor = new FunctionDescriptor(
218+
"foo",
219+
"mod.foo",
220+
parameters,
221+
false,
222+
false,
223+
Collections.emptyList(),
224+
false,
225+
null,
226+
"int | str",
227+
returnTypeDescriptor);
228+
assertDescriptorToProtobuf(functionDescriptor);
229+
}
230+
231+
@ParameterizedTest
232+
@MethodSource("typeKindTestCases")
233+
void protobufFunctionReturnTypeShouldHandleAllTypeKinds(TypeAnnotationDescriptor.TypeKind expectedTypeKind) {
234+
TypeAnnotationDescriptor returnTypeDescriptor = new TypeAnnotationDescriptor(
235+
"MyType",
236+
expectedTypeKind,
237+
List.of(),
238+
"mytype",
239+
false);
240+
List<FunctionDescriptor.Parameter> parameters = new ArrayList<>();
241+
FunctionDescriptor functionDescriptor = new FunctionDescriptor(
242+
"foo",
243+
"mod.foo",
244+
parameters,
245+
false,
246+
false,
247+
Collections.emptyList(),
248+
false,
249+
null,
250+
null,
251+
returnTypeDescriptor);
252+
assertDescriptorToProtobuf(functionDescriptor);
253+
}
254+
172255
@ParameterizedTest
173256
@MethodSource("typeKindTestCases")
174257
void protobufFunctionParameterShouldHandleAllTypeKinds(TypeAnnotationDescriptor.TypeKind expectedTypeKind) {
@@ -220,4 +303,4 @@ public static FunctionDescriptor lastFunctionDescriptor(String... code) {
220303
assertThat(functionType.definitionLocation()).contains(functionDescriptor.definitionLocation());
221304
return functionDescriptor;
222305
}
223-
}
306+
}

‎python-frontend/src/test/java/org/sonar/python/semantic/v2/TypeInferenceV2Test.java‎

Lines changed: 47 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2912,6 +2912,7 @@ void type_origin_of_project_function() {
29122912
projectLevelSymbolTable.addModule(tree, "", pythonFile("mod.py"));
29132913
ProjectLevelTypeTable projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable);
29142914

2915+
PythonType intType = projectLevelTypeTable.getBuiltinsModule().resolveMember("int").get();
29152916
var lines = """
29162917
from mod import foo
29172918
foo
@@ -2922,8 +2923,52 @@ void type_origin_of_project_function() {
29222923
FunctionType fooType = (FunctionType) ((ExpressionStatement) fileInput.statements().statements().get(1)).expressions().get(0).typeV2();
29232924
assertThat(fooType.typeOrigin()).isEqualTo(TypeOrigin.LOCAL);
29242925
PythonType xType = ((ExpressionStatement) fileInput.statements().statements().get(3)).expressions().get(0).typeV2();
2925-
// Declared return types of local functions are currently not stored in the project level symbol table
2926-
assertThat(xType).isEqualTo(PythonType.UNKNOWN);
2926+
assertThat(xType.unwrappedType()).isEqualTo(intType);
2927+
}
2928+
2929+
2930+
@Test
2931+
void type_project_function_with_self_return_type() {
2932+
FileInput tree = parseWithoutSymbols("""
2933+
from typing import Self
2934+
class A:
2935+
def foo(self) -> Self: ...
2936+
""");
2937+
ProjectLevelSymbolTable projectLevelSymbolTable = ProjectLevelSymbolTable.empty();
2938+
projectLevelSymbolTable.addModule(tree, "", pythonFile("mod.py"));
2939+
ProjectLevelTypeTable projectLevelTypeTable = new ProjectLevelTypeTable(projectLevelSymbolTable);
2940+
2941+
var lines = """
2942+
from mod import A
2943+
2944+
a = A()
2945+
a
2946+
result = a.foo()
2947+
result
2948+
""";
2949+
FileInput fileInput = inferTypes(lines, projectLevelTypeTable);
2950+
2951+
PythonType aType = ((ExpressionStatement) fileInput.statements().statements().get(2)).expressions().get(0).typeV2();
2952+
assertThat(aType).isInstanceOf(ObjectType.class);
2953+
assertThat(aType.unwrappedType()).isInstanceOf(ClassType.class);
2954+
ClassType classType = (ClassType) aType.unwrappedType();
2955+
assertThat(classType.name()).isEqualTo("A");
2956+
assertThat(classType.fullyQualifiedName()).isEqualTo("mod.A");
2957+
2958+
CallExpression callExpression = PythonTestUtils.getFirstChild(fileInput, t -> t instanceof CallExpression fooCall && fooCall.callee() instanceof QualifiedExpression);
2959+
assertThat(callExpression.callee().typeV2())
2960+
.isInstanceOfSatisfying(FunctionType.class,
2961+
functionType -> assertThat(functionType.returnType())
2962+
.isInstanceOf(ObjectType.class)
2963+
.extracting(PythonType::unwrappedType)
2964+
.isInstanceOf(SelfType.class)
2965+
.extracting(SelfType.class::cast)
2966+
.extracting(SelfType::innerType)
2967+
.isEqualTo(classType)
2968+
);
2969+
PythonType resultType = ((ExpressionStatement) fileInput.statements().statements().get(4)).expressions().get(0).typeV2();
2970+
assertThat(resultType.unwrappedType()).isInstanceOf(ClassType.class);
2971+
assertThat(resultType.unwrappedType()).isEqualTo(classType);
29272972
}
29282973

29292974
@Test

0 commit comments

Comments
 (0)