GH-2612: As a developer I want array literals to be typed as ArrayN<...> instead of just Array<union{...}> (#2613)

* add test

* first shot

* adjust tests

* add helper method

* support type inference on indexed access

* adjust test

* tweaks for ArrayN inference

* adjust tests

* next try, added tests

* next try

* adjust tests

* adjust tests

* reducer respects structural subtype check wrt. optional properties

* improved error message in case of failed poly processor

* remove obsolete casts

* adjust tests

* adjust tests

* improve nested object literals, improve non-backtracker wrt. struct. obj

* adjust tests

* fix concurrent exception

* fix hover exception

* add some tests, some src improvements

* more tests, adjust tests

* fix several issues: Deadlock during startup, Text Highlight and
Documentation, Issue creation

* adjust built-ins Array#concat to support unification of types

* add tests

* adjust tests

* more tests / corner cases

* add test, adjust tests

* support parentheses around return expressions

* adjusted tests

* several unrelated changes

* many improvements, performance, reduce unions with match, ...

* add another test for bug regarding union of methods with type vars

* enhance deadlock bugfix

* fix handling type vars of composed methods

* adjust some test expectations

* fix double reporting, adjust tests, add test case

* fix/mitigate hover bug

* postpone to GH-2615

* incorporate review changes

* bump minor version due to backwards incompatibility

Author: mmews-n4

n4js-libs/packages/n4js-cli/src/n4js/Globals.n4js

@@ -37,7 +37,7 @@ export public const JRE_INFO_MAP = new Map<Platform, JreInfo>([
     [Platform.x64_sunos,   null],
     [Platform.x32_win32,   {bin: "bin/java.exe", name: "OpenJDK-jre_x86-32_windows_hotspot_2021-05-06-23-30.zip"}],
     [Platform.x64_win32,   {bin: "bin/java.exe", name: "OpenJDK-jre_x64_windows_hotspot_2021-05-06-23-30.zip"}]]
-     as Iterable<Iterable2<Platform, JreInfo>>);
+    );
 
 
 @StringBased

n4js-libs/packages/org.eclipse.n4js.mangelhaft.test/test/n4js/org/eclipse/n4js/mangelhaft/test/TestExecutorTests.n4js

@@ -264,7 +264,7 @@ export public class TestExecutorTests {
             ["fixme__multiScope___success", TestStatus.passed],
             ["fixme__multiScope___fail", TestStatus.failed],
         ]
-        const expectedMap = new Map<string, string>(expected as Iterable<Iterable2<string,string>>);
+        const expectedMap = new Map<string, string>(expected);
         const tests: InstrumentedTest[] = []        
         tests.push(new InstrumentedTest().load(FixmeTests).setTestObject(new FixmeTests()));
         const val =  await this.subject.executor.runTestsAsync(tests);
@@ -286,7 +286,7 @@ export public class TestExecutorTests {
             ["fixme__multiScope___success", TestStatus.failed],
             ["fixme__multiScope___fail", TestStatus.skipped_fixme],
         ]
-        const expectedMap = new Map<string, string>(expected as Iterable<Iterable2<string,string>>);
+        const expectedMap = new Map<string, string>(expected);
         const tests: InstrumentedTest[] = []        
         tests.push(new InstrumentedTest().load(FixmeTests).setTestObject(new FixmeTests()));
         const val =  await this.subject.executor.runTestsAsync(tests, "SERVER");
@@ -308,7 +308,7 @@ export public class TestExecutorTests {
             ["fixme__multiScope___success", TestStatus.failed],
             ["fixme__multiScope___fail", TestStatus.skipped_fixme],
         ]
-        const expectedMap = new Map<string, string>(expected as Iterable<Iterable2<string,string>>);
+        const expectedMap = new Map<string, string>(expected);
         const tests: InstrumentedTest[] = []        
         tests.push(new InstrumentedTest().load(FixmeTests).setTestObject(new FixmeTests()));
         const val =  await this.subject.executor.runTestsAsync(tests, "MOCK");

n4js-libs/packages/org.eclipse.n4js.mangelhaft/src/n4js/org/eclipse/n4js/mangelhaft/InstrumentedTest.n4js

@@ -155,7 +155,7 @@ export public class InstrumentedTest implements IInstrumentedTest {
         // Make shared TestController instance be accessible to test instances (via injection):
         const instance = testInjector.internalCreate(testClass, testInjector, new Map<string, N4Object>([
                 [`${TestController.n4type.origin}${TestController.n4type.fqn}`, controller]
-            ] as Iterable<Iterable2<string, N4Object>>));
+            ]));
 
         return new InstrumentedTest(testClass, info, instance, null, parameterizedTests);
     }

n4js-libs/packages/org.eclipse.n4js.mangelhaft/src/n4js/org/eclipse/n4js/mangelhaft/types/TestStatus.n4js

@@ -28,7 +28,7 @@ let testStatusIntMap = new Map<TestStatus, int>([
   , [TestStatus.failed, 6]
   , [TestStatus.error, 7]
 
-] as Iterable<Iterable2<TestStatus,int>>);
+]);
 
 export function aggregateTestStatuses(testStatus1: TestStatus, testStatus2: TestStatus): TestStatus {
     return testStatusIntMap.get(testStatus1) > testStatusIntMap.get(testStatus2) ? testStatus1 : testStatus2;

plugins/org.eclipse.n4js.ide/src/org/eclipse/n4js/ide/server/N4JSResourceTaskContext.java

@@ -13,8 +13,13 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import org.eclipse.emf.common.util.URI;
 import org.eclipse.n4js.resource.N4JSResource;
+import org.eclipse.n4js.scoping.builtin.BuiltInTypeScope;
 import org.eclipse.n4js.xtext.ide.server.ResourceTaskContext;
+import org.eclipse.n4js.xtext.ide.server.ResourceTaskManager;
+import org.eclipse.n4js.xtext.ide.server.util.XChunkedResourceDescriptions;
+import org.eclipse.n4js.xtext.workspace.WorkspaceConfigSnapshot;
 import org.eclipse.xtext.resource.XtextResource;
 
 /**
@@ -29,4 +34,15 @@ public class N4JSResourceTaskContext extends ResourceTaskContext {
 		return options;
 	}
 
+	@Override
+	public synchronized void initialize(
+			ResourceTaskManager parent,
+			URI uri,
+			boolean isTemporary,
+			XChunkedResourceDescriptions index,
+			WorkspaceConfigSnapshot workspaceConfig) {
+
+		super.initialize(parent, uri, isTemporary, index, workspaceConfig);
+		BuiltInTypeScope.get(getResourceSet()).getNullType(); // force loading built-ins to avoid deadlock later
+	}
 }

plugins/org.eclipse.n4js.ide/src/org/eclipse/n4js/ide/server/symbol/N4JSDocumentSymbolService.java

@@ -109,7 +109,9 @@ public List<? extends Location> getDefinitions(XtextResource resource, int offse
 						.eGet(N4JSPackage.eINSTANCE.getPropertyNameValuePair_Expression())) {
 
 			EObject elementInDestructuring = findReferenceHelper.getMemberInDestructuring(elemAtOffset);
-			addLocations(resourceAccess, cancelIndicator, elementInDestructuring, locations);
+			if (elementInDestructuring != null) {
+				addLocations(resourceAccess, cancelIndicator, elementInDestructuring, locations);
+			}
 		}
 
 		addLocations(resourceAccess, cancelIndicator, element, locations);

plugins/org.eclipse.n4js.model/src/org/eclipse/n4js/n4JS/N4JSASTUtils.java

@@ -346,12 +346,17 @@ public static FunctionOrFieldAccessor getContainingFunctionOrAccessor(EObject eo
 	 * arrow function; otherwise <code>null</code> is returned.
 	 */
 	public static ArrowFunction getContainingSingleExpressionArrowFunction(Expression expression) {
-		final EObject parent = expression.eContainer();
+		EObject skippedParenthesesExpr = expression;
+		EObject parent = expression.eContainer();
+		while (parent instanceof ParenExpression) {
+			skippedParenthesesExpr = parent;
+			parent = parent.eContainer();
+		}
 		final EObject grandparent = parent != null ? parent.eContainer() : null;
 		final EObject grandgrandparent = grandparent != null ? grandparent.eContainer() : null;
 		if (grandgrandparent instanceof ArrowFunction) {
 			final ArrowFunction arrFun = (ArrowFunction) grandgrandparent;
-			if (arrFun.isSingleExprImplicitReturn() && arrFun.implicitReturnExpr() == expression) {
+			if (arrFun.isSingleExprImplicitReturn() && arrFun.implicitReturnExpr() == skippedParenthesesExpr) {
 				return arrFun;
 			}
 		}

plugins/org.eclipse.n4js.smith/src/org/eclipse/n4js/smith/N4JSDataCollectors.java

@@ -72,6 +72,11 @@ public final class N4JSDataCollectors {
 	public static final DataCollector dcCacheMakeKeys = create("MakeKeys", dcCache);
 	public static final DataCollector dcCacheSetTarget = create("SetTarget", dcCache);
 
+	public static final DataCollector dcInferenceContextSolve = create("Solve");
+	public static final DataCollector dcInferenceContextReduce = create("Reduce", dcInferenceContextSolve);
+	public static final DataCollector dcInferenceContextIncorporate = create("Incorporate", dcInferenceContextSolve);
+	public static final DataCollector dcInferenceContextResolve = create("Resolve", dcInferenceContextSolve);
+
 	public static final DataCollector dcTypeHierachyTraverser = create("TypeHierachyTraverser");
 	public static final DataCollector dcTHT_AllMembersCollector = create("AllMembersCollector",
 			dcTypeHierachyTraverser);

plugins/org.eclipse.n4js.xtext.ide/src/org/eclipse/n4js/xtext/ide/server/TextDocumentFrontend.java

@@ -356,7 +356,11 @@ protected List<? extends DocumentHighlight> documentHighlight(ResourceTaskContex
 		}
 		XtextResource res = rtc.getResource();
 		XDocument doc = rtc.getDocument();
-		return service.getDocumentHighlights(doc, res, params, cancelIndicator);
+		try {
+			return service.getDocumentHighlights(doc, res, params, cancelIndicator);
+		} catch (Exception e) {
+			return Collections.emptyList();
+		}
 	}
 
 	@Override

plugins/org.eclipse.n4js/src-env/env/builtin_js.n4jsd

@@ -377,7 +377,7 @@ export external public class Array<T> {
 	public toLocaleString(): string;
 	public pop(): T;
 	public push(...items: T): number;
-	public concat(...items: union {T, Array<? extends T>}): Array<T>;
+	public <S>concat(...items: union {S, Array<? extends S>}): Array<union{S,T}>;
 	public join(separator: string = ): string;
 	public reverse(): Array<T>;
 	public shift(): T;

plugins/org.eclipse.n4js/src/org/eclipse/n4js/naming/N4JSImportedNamesAdapter.java

@@ -10,17 +10,18 @@
  */
 package org.eclipse.n4js.naming;
 
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
+import java.util.TreeSet;
 
 import org.eclipse.xtext.linking.impl.ImportedNamesAdapter;
 import org.eclipse.xtext.naming.QualifiedName;
 import org.eclipse.xtext.resource.IEObjectDescription;
 import org.eclipse.xtext.scoping.IScope;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
 
 /**
  * Adapts default implementation to not normalize qualified names to be all lower case.
@@ -72,18 +73,18 @@ public IScope wrap(IScope scope) {
 	}
 
 	@Override
-	public Set<QualifiedName> getImportedNames() {
-		return Collections.unmodifiableSet(nonNullImportedNames);
+	synchronized public TreeSet<QualifiedName> getImportedNames() {
+		return Sets.newTreeSet(nonNullImportedNames);
 	}
 
 	/** Adds given element to set of imported names. */
-	public void addImportedName(QualifiedName name) {
+	synchronized public void addImportedName(QualifiedName name) {
 		Preconditions.checkNotNull(name);
 		nonNullImportedNames.add(name);
 	}
 
 	@Override
-	public void clear() {
+	synchronized public void clear() {
 		nonNullImportedNames.clear();
 	}
 }

plugins/org.eclipse.n4js/src/org/eclipse/n4js/postprocessing/AbstractPolyProcessor.java

@@ -17,7 +17,10 @@
 import static org.eclipse.xtext.xbase.lib.IterableExtensions.toSet;
 import static org.eclipse.xtext.xbase.lib.IteratorExtensions.exists;
 
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -39,6 +42,7 @@
 import org.eclipse.n4js.n4JS.PropertyNameValuePair;
 import org.eclipse.n4js.n4JS.PropertySetterDeclaration;
 import org.eclipse.n4js.n4JS.PropertySpread;
+import org.eclipse.n4js.n4JS.ReturnStatement;
 import org.eclipse.n4js.ts.typeRefs.FunctionTypeExprOrRef;
 import org.eclipse.n4js.ts.typeRefs.TypeRef;
 import org.eclipse.n4js.ts.types.InferenceVariable;
@@ -56,6 +60,7 @@
 import org.eclipse.n4js.typesystem.utils.TypeSystemHelper;
 import org.eclipse.n4js.typesystem.utils.TypeSystemHelper.Callable;
 import org.eclipse.n4js.utils.N4JSLanguageUtils;
+import org.eclipse.xtext.xbase.lib.IteratorExtensions;
 
 import com.google.inject.Inject;
 
@@ -294,6 +299,26 @@ protected Map<InferenceVariable, TypeRef> createPseudoSolution(InferenceContext
 		return pseudoSolution;
 	}
 
+	protected List<Expression> getReturnExpressions(FunctionDefinition fun) {
+		if (fun instanceof ArrowFunction && ((ArrowFunction) fun).isSingleExprImplicitReturn()) {
+			Expression singleExpression = ((ArrowFunction) fun).getSingleExpression();
+			if (singleExpression != null) {
+				return Collections.singletonList(singleExpression);
+			}
+		}
+		if (fun.getBody() != null) {
+			List<ReturnStatement> returnStmts = IteratorExtensions.toList(fun.getBody().getAllReturnStatements());
+			List<Expression> returnExpr = new ArrayList<>();
+			for (ReturnStatement rs : returnStmts) {
+				if (rs.getExpression() != null) {
+					returnExpr.add(rs.getExpression());
+				}
+			}
+			return returnExpr;
+		}
+		return Collections.emptyList();
+	}
+
 	// FIXME move to a better place
 	protected boolean isReturningValue(FunctionDefinition fun) {
 		return (fun.getBody() != null && exists(fun.getBody().getAllReturnStatements(), s -> s.getExpression() != null))

plugins/org.eclipse.n4js/src/org/eclipse/n4js/postprocessing/PolyProcessor.java

@@ -20,11 +20,13 @@
 import org.eclipse.n4js.n4JS.Expression;
 import org.eclipse.n4js.n4JS.FormalParameter;
 import org.eclipse.n4js.n4JS.FunctionExpression;
+import org.eclipse.n4js.n4JS.N4JSASTUtils;
 import org.eclipse.n4js.n4JS.ObjectLiteral;
 import org.eclipse.n4js.n4JS.ParameterizedCallExpression;
 import org.eclipse.n4js.n4JS.PropertyAssignment;
 import org.eclipse.n4js.n4JS.PropertyMethodDeclaration;
 import org.eclipse.n4js.n4JS.RelationalExpression;
+import org.eclipse.n4js.n4JS.YieldExpression;
 import org.eclipse.n4js.ts.typeRefs.TypeRef;
 import org.eclipse.n4js.ts.types.TypableElement;
 import org.eclipse.n4js.ts.types.util.Variance;
@@ -161,14 +163,14 @@ void inferType(RuleEnvironment G, Expression rootPoly, ASTMetaInfoCache cache) {
 			infCtx.addConstraint(TypeConstraint.FALSE);
 		}
 
-		TypeRef expectedTypeOfPoly = destructureHelper.calculateExpectedType(rootPoly, G, infCtx);
+		TypeRef expectedTypeByLiteralStructure = destructureHelper.calculateExpectedType(rootPoly, G, infCtx);
 		// we have to pass the expected type to the #getType() method, so retrieve it first
 		// (until the expectedType judgment is integrated into AST traversal, we have to invoke this judgment here;
 		// in case of not-well-behaving expectedType rules, we use 'null' as expected type, i.e. no expectation)
 		// TODO integrate expectedType judgment into AST traversal and remove #isProblematicCaseOfExpectedType()
 		TypeRef expectedTypeRef = null;
-		if (expectedTypeOfPoly != null) {
-			expectedTypeRef = expectedTypeOfPoly;
+		if (expectedTypeByLiteralStructure != null) {
+			expectedTypeRef = expectedTypeByLiteralStructure;
 		} else if (!isProblematicCaseOfExpectedType(rootPoly)) {
 			expectedTypeRef = ts.expectedType(G, rootPoly.eContainer(), rootPoly);
 		}
@@ -261,7 +263,16 @@ protected TypeRef processExpr(RuleEnvironment G, Expression expr, TypeRef expect
 	 * Returns true if we are not allowed to ask for the expected type of 'node', because this would lead to illegal
 	 * forward references (temporary).
 	 */
-	private boolean isProblematicCaseOfExpectedType(EObject node) {
-		return node != null && node.eContainer() instanceof RelationalExpression;
+	private boolean isProblematicCaseOfExpectedType(Expression node) {
+		if (node != null) {
+			EObject parent = N4JSASTUtils.skipParenExpressionUpward(node.eContainer());
+			if (parent instanceof RelationalExpression) {
+				return true;
+			}
+			if (parent instanceof YieldExpression) {
+				return true;
+			}
+		}
+		return false;
 	}
 }