Skip to content

Commit 19df9a2

Browse files
janedbaljanedbal
authored
Collected data: reduce memory consumption & result cache size
1 parent ef6cc0a commit 19df9a2

11 files changed

+69
-43
lines changed

Diff for: ‎src/Analyser/Analyser.php

+4-1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212
use function count;
1313
use function memory_get_peak_usage;
1414

15+
/**
16+
* @phpstan-import-type CollectorData from CollectedData
17+
*/
1518
final class Analyser
1619
{
1720

@@ -59,7 +62,7 @@ public function analyse(
5962
$linesToIgnore = [];
6063
$unmatchedLineIgnores = [];
6164

62-
/** @var list<CollectedData> $collectedData */
65+
/** @var CollectorData $collectedData */
6366
$collectedData = [];
6467

6568
$internalErrorsCount = 0;

Diff for: ‎src/Analyser/AnalyserResult.php

+3-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
/**
1010
* @phpstan-import-type LinesToIgnore from FileAnalyserResult
11+
* @phpstan-import-type CollectorData from CollectedData
1112
*/
1213
final class AnalyserResult
1314
{
@@ -22,7 +23,7 @@ final class AnalyserResult
2223
* @param list<Error> $locallyIgnoredErrors
2324
* @param array<string, LinesToIgnore> $linesToIgnore
2425
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
25-
* @param list<CollectedData> $collectedData
26+
* @param CollectorData $collectedData
2627
* @param list<InternalError> $internalErrors
2728
* @param array<string, array<string>>|null $dependencies
2829
* @param array<string, array<RootExportedNode>> $exportedNodes
@@ -125,7 +126,7 @@ public function getInternalErrors(): array
125126
}
126127

127128
/**
128-
* @return list<CollectedData>
129+
* @return CollectorData
129130
*/
130131
public function getCollectedData(): array
131132
{

Diff for: ‎src/Analyser/FileAnalyser.php

+5-6
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@
3737
use const E_USER_WARNING;
3838
use const E_WARNING;
3939

40+
/**
41+
* @phpstan-import-type CollectorData from CollectedData
42+
*/
4043
final class FileAnalyser
4144
{
4245

@@ -76,7 +79,7 @@ public function analyseFile(
7679
/** @var list<Error> $locallyIgnoredErrors */
7780
$locallyIgnoredErrors = [];
7881

79-
/** @var list<CollectedData> $fileCollectedData */
82+
/** @var CollectorData $fileCollectedData */
8083
$fileCollectedData = [];
8184

8285
$fileDependencies = [];
@@ -195,11 +198,7 @@ public function analyseFile(
195198
continue;
196199
}
197200

198-
$fileCollectedData[] = new CollectedData(
199-
$collectedData,
200-
$scope->getFile(),
201-
get_class($collector),
202-
);
201+
$fileCollectedData[$scope->getFile()][get_class($collector)][] = $collectedData;
203202
}
204203

205204
try {

Diff for: ‎src/Analyser/FileAnalyserResult.php

+3-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
/**
99
* @phpstan-type LinesToIgnore = array<string, array<int, non-empty-list<string>|null>>
10+
* @phpstan-import-type CollectorData from CollectedData
1011
*/
1112
final class FileAnalyserResult
1213
{
@@ -16,7 +17,7 @@ final class FileAnalyserResult
1617
* @param list<Error> $filteredPhpErrors
1718
* @param list<Error> $allPhpErrors
1819
* @param list<Error> $locallyIgnoredErrors
19-
* @param list<CollectedData> $collectedData
20+
* @param CollectorData $collectedData
2021
* @param list<string> $dependencies
2122
* @param list<RootExportedNode> $exportedNodes
2223
* @param LinesToIgnore $linesToIgnore
@@ -69,7 +70,7 @@ public function getLocallyIgnoredErrors(): array
6970
}
7071

7172
/**
72-
* @return list<CollectedData>
73+
* @return CollectorData
7374
*/
7475
public function getCollectedData(): array
7576
{

Diff for: ‎src/Analyser/ResultCache/ResultCache.php

+3-2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
/**
1111
* @phpstan-import-type LinesToIgnore from FileAnalyserResult
12+
* @phpstan-import-type CollectorData from CollectedData
1213
*/
1314
final class ResultCache
1415
{
@@ -20,7 +21,7 @@ final class ResultCache
2021
* @param array<string, list<Error>> $locallyIgnoredErrors
2122
* @param array<string, LinesToIgnore> $linesToIgnore
2223
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
23-
* @param array<string, array<CollectedData>> $collectedData
24+
* @param CollectorData $collectedData
2425
* @param array<string, array<string>> $dependencies
2526
* @param array<string, array<RootExportedNode>> $exportedNodes
2627
* @param array<string, array{string, bool, string}> $projectExtensionFiles
@@ -101,7 +102,7 @@ public function getUnmatchedLineIgnores(): array
101102
}
102103

103104
/**
104-
* @return array<string, array<CollectedData>>
105+
* @return CollectorData
105106
*/
106107
public function getCollectedData(): array
107108
{

Diff for: ‎src/Analyser/ResultCache/ResultCacheManager.php

+10-15
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949

5050
/**
5151
* @phpstan-import-type LinesToIgnore from FileAnalyserResult
52+
* @phpstan-import-type CollectorData from CollectedData
5253
*/
5354
final class ResultCacheManager
5455
{
@@ -406,10 +407,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
406407
$freshLocallyIgnoredErrorsByFile[$error->getFilePath()][] = $error;
407408
}
408409

409-
$freshCollectedDataByFile = [];
410-
foreach ($analyserResult->getCollectedData() as $collectedData) {
411-
$freshCollectedDataByFile[$collectedData->getFilePath()][] = $collectedData;
412-
}
410+
$freshCollectedDataByFile = $analyserResult->getCollectedData();
413411

414412
$meta = $resultCache->getMeta();
415413
$projectConfigArray = $meta['projectConfig'];
@@ -524,13 +522,6 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
524522
}
525523
}
526524

527-
$flatCollectedData = [];
528-
foreach ($collectedDataByFile as $fileCollectedData) {
529-
foreach ($fileCollectedData as $collectedData) {
530-
$flatCollectedData[] = $collectedData;
531-
}
532-
}
533-
534525
return new ResultCacheProcessResult(new AnalyserResult(
535526
$flatErrors,
536527
$analyserResult->getFilteredPhpErrors(),
@@ -539,7 +530,7 @@ public function process(AnalyserResult $analyserResult, ResultCache $resultCache
539530
$linesToIgnore,
540531
$unmatchedLineIgnores,
541532
$internalErrors,
542-
$flatCollectedData,
533+
$collectedDataByFile,
543534
$dependencies,
544535
$exportedNodes,
545536
$analyserResult->hasReachedInternalErrorsCountLimit(),
@@ -584,8 +575,8 @@ private function mergeLocallyIgnoredErrors(ResultCache $resultCache, array $fres
584575
}
585576

586577
/**
587-
* @param array<string, array<CollectedData>> $freshCollectedDataByFile
588-
* @return array<string, array<CollectedData>>
578+
* @param CollectorData $freshCollectedDataByFile
579+
* @return CollectorData
589580
*/
590581
private function mergeCollectedData(ResultCache $resultCache, array $freshCollectedDataByFile): array
591582
{
@@ -704,7 +695,7 @@ private function mergeUnmatchedLineIgnores(ResultCache $resultCache, array $fres
704695
* @param array<string, list<Error>> $locallyIgnoredErrors
705696
* @param array<string, LinesToIgnore> $linesToIgnore
706697
* @param array<string, LinesToIgnore> $unmatchedLineIgnores
707-
* @param array<string, array<CollectedData>> $collectedData
698+
* @param array<string, array<string, list<CollectedData>>> $collectedData
708699
* @param array<string, array<string>> $dependencies
709700
* @param array<string, array<RootExportedNode>> $exportedNodes
710701
* @param array<string, array{string, bool, string}> $projectExtensionFiles
@@ -760,6 +751,10 @@ private function save(
760751
ksort($collectedData);
761752
ksort($invertedDependencies);
762753

754+
foreach ($collectedData as & $collectedDataPerFile) {
755+
ksort($collectedDataPerFile);
756+
}
757+
763758
foreach ($invertedDependencies as $file => $fileData) {
764759
$dependentFiles = $fileData['dependentFiles'];
765760
sort($dependentFiles);

Diff for: ‎src/Collectors/CollectedData.php

+2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88

99
/**
1010
* @api
11+
*
12+
* @phpstan-type CollectorData = array<string, array<class-string<Collector<Node, mixed>>, list<mixed>>>
1113
*/
1214
final class CollectedData implements JsonSerializable
1315
{

Diff for: ‎src/Command/AnalyseApplication.php

+21-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
use PHPStan\Analyser\AnalyserResultFinalizer;
77
use PHPStan\Analyser\Ignore\IgnoredErrorHelper;
88
use PHPStan\Analyser\ResultCache\ResultCacheManagerFactory;
9+
use PHPStan\Collectors\CollectedData;
910
use PHPStan\Internal\BytesHelper;
1011
use PHPStan\PhpDoc\StubFilesProvider;
1112
use PHPStan\PhpDoc\StubValidator;
@@ -19,6 +20,9 @@
1920
use function sha1_file;
2021
use function sprintf;
2122

23+
/**
24+
* @phpstan-import-type CollectorData from CollectedData
25+
*/
2226
final class AnalyseApplication
2327
{
2428

@@ -150,7 +154,7 @@ public function analyse(
150154
$notFileSpecificErrors,
151155
$internalErrors,
152156
[],
153-
$collectedData,
157+
$this->mapCollectedData($collectedData),
154158
$defaultLevelUsed,
155159
$projectConfigFile,
156160
$savedResultCache,
@@ -160,6 +164,22 @@ public function analyse(
160164
);
161165
}
162166

167+
/**
168+
* @param CollectorData $collectedData
169+
*
170+
* @return list<CollectedData>
171+
*/
172+
private function mapCollectedData(array $collectedData): array
173+
{
174+
$result = [];
175+
foreach ($collectedData as $file => $dataPerCollector) {
176+
foreach ($dataPerCollector as $collectorType => $rawData) {
177+
$result[] = new CollectedData($rawData, $file, $collectorType);
178+
}
179+
}
180+
return $result;
181+
}
182+
163183
/**
164184
* @param string[] $files
165185
* @param string[] $allAnalysedFiles

Diff for: ‎src/Command/WorkerCommand.php

+6-2
Original file line numberDiff line numberDiff line change
@@ -226,8 +226,12 @@ private function runWorker(
226226
foreach ($fileAnalyserResult->getLocallyIgnoredErrors() as $locallyIgnoredError) {
227227
$locallyIgnoredErrors[] = $locallyIgnoredError;
228228
}
229-
foreach ($fileAnalyserResult->getCollectedData() as $data) {
230-
$collectedData[] = $data;
229+
foreach ($fileAnalyserResult->getCollectedData() as $collectedFile => $dataPerCollector) {
230+
foreach ($dataPerCollector as $collectorType => $collectorData) {
231+
foreach ($collectorData as $data) {
232+
$collectedData[$collectedFile][$collectorType][] = $data;
233+
}
234+
}
231235
}
232236
} catch (Throwable $t) {
233237
$internalErrorsCount++;

Diff for: ‎src/Node/CollectedDataNode.php

+6-9
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,16 @@
66
use PhpParser\NodeAbstract;
77
use PHPStan\Collectors\CollectedData;
88
use PHPStan\Collectors\Collector;
9-
use function array_key_exists;
109

1110
/**
1211
* @api
12+
* @phpstan-import-type CollectorData from CollectedData
1313
*/
1414
final class CollectedDataNode extends NodeAbstract implements VirtualNode
1515
{
1616

1717
/**
18-
* @param CollectedData[] $collectedData
18+
* @param CollectorData $collectedData
1919
*/
2020
public function __construct(private array $collectedData, private bool $onlyFiles)
2121
{
@@ -31,17 +31,14 @@ public function __construct(private array $collectedData, private bool $onlyFile
3131
public function get(string $collectorType): array
3232
{
3333
$result = [];
34-
foreach ($this->collectedData as $collectedData) {
35-
if ($collectedData->getCollectorType() !== $collectorType) {
34+
foreach ($this->collectedData as $filePath => $collectedDataPerCollector) {
35+
if (!isset($collectedDataPerCollector[$collectorType])) {
3636
continue;
3737
}
3838

39-
$filePath = $collectedData->getFilePath();
40-
if (!array_key_exists($filePath, $result)) {
41-
$result[$filePath] = [];
39+
foreach ($collectedDataPerCollector[$collectorType] as $rawData) {
40+
$result[$filePath][] = $rawData;
4241
}
43-
44-
$result[$filePath][] = $collectedData->getData();
4542
}
4643

4744
return $result;

Diff for: ‎src/Parallel/ParallelAnalyser.php

+6-3
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
use PHPStan\Analyser\AnalyserResult;
1010
use PHPStan\Analyser\Error;
1111
use PHPStan\Analyser\InternalError;
12-
use PHPStan\Collectors\CollectedData;
1312
use PHPStan\Dependency\RootExportedNode;
1413
use PHPStan\Process\ProcessHelper;
1514
use React\EventLoop\LoopInterface;
@@ -211,8 +210,12 @@ public function analyse(
211210
$locallyIgnoredErrors[] = $locallyIgnoredFileError;
212211
}
213212

214-
foreach ($json['collectedData'] as $jsonData) {
215-
$collectedData[] = CollectedData::decode($jsonData);
213+
foreach ($json['collectedData'] as $file => $jsonDataByCollector) {
214+
foreach ($jsonDataByCollector as $collectorType => $listOfCollectedData) {
215+
foreach ($listOfCollectedData as $rawCollectedData) {
216+
$collectedData[$file][$collectorType][] = $rawCollectedData;
217+
}
218+
}
216219
}
217220

218221
/**

0 commit comments

Comments
 (0)