Skip to content

Commit 7e6102e

Browse files
zepfredzepfred
authored
feat: Improve ScoreAnalysis debug information (#923)
1 parent 6b9ac28 commit 7e6102e

File tree

6 files changed

+433
-24
lines changed

6 files changed

+433
-24
lines changed

‎core/src/main/java/ai/timefold/solver/core/api/score/analysis/ConstraintAnalysis.java

Lines changed: 71 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
package ai.timefold.solver.core.api.score.analysis;
22

3+
import static ai.timefold.solver.core.api.score.analysis.ScoreAnalysis.DEFAULT_SUMMARY_CONSTRAINT_MATCH_LIMIT;
4+
import static java.util.Comparator.comparing;
5+
6+
import java.util.Comparator;
37
import java.util.HashSet;
48
import java.util.List;
59
import java.util.Map;
610
import java.util.Objects;
7-
import java.util.stream.Collectors;
811
import java.util.stream.Stream;
912

1013
import ai.timefold.solver.core.api.score.Score;
@@ -56,6 +59,21 @@ static <Score_ extends Score<Score_>> ConstraintAnalysis<Score_> of(ConstraintRe
5659
Objects.requireNonNull(score);
5760
}
5861

62+
/**
63+
* Return the match count of the constraint.
64+
*
65+
* @throws IllegalStateException if the {@link ConstraintAnalysis#matches()} is null
66+
*/
67+
public int matchCount() {
68+
if (matches == null) {
69+
throw new IllegalArgumentException("""
70+
The constraint matches must be non-null.
71+
Maybe use ScoreAnalysisFetchPolicy.FETCH_ALL to request the score analysis
72+
""");
73+
}
74+
return matches.size();
75+
}
76+
5977
ConstraintAnalysis<Score_> negate() {
6078
if (matches == null) {
6179
return ConstraintAnalysis.of(constraintRef, weight.negate(), score.negate());
@@ -72,9 +90,9 @@ static <Score_ extends Score<Score_>> ConstraintAnalysis<Score_> diff(
7290
ConstraintAnalysis<Score_> otherConstraintAnalysis) {
7391
if (constraintAnalysis == null) {
7492
if (otherConstraintAnalysis == null) {
75-
throw new IllegalStateException("""
76-
Impossible state: none of the score explanations provided constraint matches for a constraint (%s).
77-
""".formatted(constraintRef));
93+
throw new IllegalStateException(
94+
"Impossible state: none of the score explanations provided constraint matches for a constraint (%s)."
95+
.formatted(constraintRef));
7896
}
7997
// No need to compute diff; this constraint is not present in this score explanation.
8098
return otherConstraintAnalysis.negate();
@@ -85,9 +103,9 @@ static <Score_ extends Score<Score_>> ConstraintAnalysis<Score_> diff(
85103
var matchAnalyses = constraintAnalysis.matches();
86104
var otherMatchAnalyses = otherConstraintAnalysis.matches();
87105
if ((matchAnalyses == null && otherMatchAnalyses != null) || (matchAnalyses != null && otherMatchAnalyses == null)) {
88-
throw new IllegalStateException("""
89-
Impossible state: Only one of the score analyses (%s, %s) provided match analyses for a constraint (%s)."""
90-
.formatted(constraintAnalysis, otherConstraintAnalysis, constraintRef));
106+
throw new IllegalStateException(
107+
"Impossible state: Only one of the score analyses (%s, %s) provided match analyses for a constraint (%s)."
108+
.formatted(constraintAnalysis, otherConstraintAnalysis, constraintRef));
91109
}
92110
// Compute the diff.
93111
var constraintWeightDifference = constraintAnalysis.weight().subtract(otherConstraintAnalysis.weight());
@@ -104,9 +122,9 @@ static <Score_ extends Score<Score_>> ConstraintAnalysis<Score_> diff(
104122
var otherMatchAnalysis = otherMatchAnalysisMap.get(justification);
105123
if (matchAnalysis == null) {
106124
if (otherMatchAnalysis == null) {
107-
throw new IllegalStateException("""
108-
Impossible state: none of the match analyses provided for a constraint (%s).
109-
""".formatted(constraintRef));
125+
throw new IllegalStateException(
126+
"Impossible state: none of the match analyses provided for a constraint (%s)."
127+
.formatted(constraintRef));
110128
}
111129
// No need to compute diff; this match is not present in this score explanation.
112130
return otherMatchAnalysis.negate();
@@ -118,7 +136,7 @@ static <Score_ extends Score<Score_>> ConstraintAnalysis<Score_> diff(
118136
justification);
119137
}
120138
})
121-
.collect(Collectors.toList());
139+
.toList();
122140
return new ConstraintAnalysis<>(constraintRef, constraintWeightDifference, scoreDifference, result);
123141
}
124142

@@ -156,6 +174,48 @@ public String constraintName() {
156174
return constraintRef.constraintName();
157175
}
158176

177+
/**
178+
* Returns a diagnostic text that explains part of the score quality through the {@link ConstraintAnalysis} API.
179+
* The string is built fresh every time the method is called.
180+
*
181+
* @return never null
182+
*/
183+
@SuppressWarnings("java:S3457")
184+
public String summarize() {
185+
var summary = new StringBuilder();
186+
summary.append("""
187+
Explanation of score (%s):
188+
Constraint matches:
189+
""".formatted(score));
190+
Comparator<MatchAnalysis<Score_>> matchScoreComparator = comparing(MatchAnalysis::score);
191+
192+
var constraintMatches = matches();
193+
if (constraintMatches == null) {
194+
throw new IllegalArgumentException("""
195+
The constraint matches must be non-null.
196+
Maybe use ScoreAnalysisFetchPolicy.FETCH_ALL to request the score analysis
197+
""");
198+
}
199+
if (constraintMatches.isEmpty()) {
200+
summary.append(
201+
"%8s%s: constraint (%s) has no matches.\n".formatted(" ", score().toShortString(),
202+
constraintRef().constraintName()));
203+
} else {
204+
summary.append("%8s%s: constraint (%s) has %s matches:\n".formatted(" ", score().toShortString(),
205+
constraintRef().constraintName(), constraintMatches.size()));
206+
}
207+
constraintMatches.stream()
208+
.sorted(matchScoreComparator)
209+
.limit(DEFAULT_SUMMARY_CONSTRAINT_MATCH_LIMIT)
210+
.forEach(match -> summary.append("%12S%s: justified with (%s)\n".formatted(" ", match.score().toShortString(),
211+
match.justification())));
212+
if (constraintMatches.size() > DEFAULT_SUMMARY_CONSTRAINT_MATCH_LIMIT) {
213+
summary.append("%12s%s\n".formatted(" ", "..."));
214+
}
215+
216+
return summary.toString();
217+
}
218+
159219
@Override
160220
public String toString() {
161221
if (matches == null) {

‎core/src/main/java/ai/timefold/solver/core/api/score/analysis/ScoreAnalysis.java

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package ai.timefold.solver.core.api.score.analysis;
22

3+
import static java.util.Comparator.comparing;
4+
35
import java.util.Collection;
46
import java.util.Collections;
57
import java.util.Comparator;
@@ -53,6 +55,8 @@
5355
public record ScoreAnalysis<Score_ extends Score<Score_>>(Score_ score,
5456
Map<ConstraintRef, ConstraintAnalysis<Score_>> constraintMap) {
5557

58+
static final int DEFAULT_SUMMARY_CONSTRAINT_MATCH_LIMIT = 3;
59+
5660
public ScoreAnalysis {
5761
Objects.requireNonNull(score, "score");
5862
Objects.requireNonNull(constraintMap, "constraintMap");
@@ -141,4 +145,70 @@ public Collection<ConstraintAnalysis<Score_>> constraintAnalyses() {
141145
return constraintMap.values();
142146
}
143147

148+
/**
149+
* Returns a diagnostic text that explains the solution through the {@link ConstraintAnalysis} API to identify which
150+
* constraints cause that score quality.
151+
* The string is built fresh every time the method is called.
152+
* <p>
153+
* In case of an {@link Score#isFeasible() infeasible} solution, this can help diagnose the cause of that.
154+
*
155+
* <p>
156+
* Do not parse the return value, its format may change without warning.
157+
* Instead, provide this information in a UI or a service,
158+
* use {@link ScoreAnalysis#constraintAnalyses()}
159+
* and convert those into a domain-specific API.
160+
*
161+
* @return never null
162+
*/
163+
@SuppressWarnings("java:S3457")
164+
public String summarize() {
165+
StringBuilder summary = new StringBuilder();
166+
summary.append("""
167+
Explanation of score (%s):
168+
Constraint matches:
169+
""".formatted(score));
170+
Comparator<ConstraintAnalysis<Score_>> constraintsScoreComparator = comparing(ConstraintAnalysis::score);
171+
Comparator<MatchAnalysis<Score_>> matchScoreComparator = comparing(MatchAnalysis::score);
172+
173+
constraintAnalyses().stream()
174+
.sorted(constraintsScoreComparator)
175+
.forEach(constraint -> {
176+
var matches = constraint.matches();
177+
if (matches == null) {
178+
throw new IllegalArgumentException("""
179+
The constraint matches must be non-null.
180+
Maybe use ScoreAnalysisFetchPolicy.FETCH_ALL to request the score analysis
181+
""");
182+
}
183+
if (matches.isEmpty()) {
184+
summary.append(
185+
"%8s%s: constraint (%s) has no matches.\n".formatted(" ", constraint.score().toShortString(),
186+
constraint.constraintRef().constraintName()));
187+
} else {
188+
summary.append(
189+
"%8s%s: constraint (%s) has %s matches:\n".formatted(" ", constraint.score().toShortString(),
190+
constraint.constraintRef().constraintName(), matches.size()));
191+
}
192+
matches.stream()
193+
.sorted(matchScoreComparator)
194+
.limit(DEFAULT_SUMMARY_CONSTRAINT_MATCH_LIMIT)
195+
.forEach(match -> summary
196+
.append("%12s%s: justified with (%s)\n".formatted(" ", match.score().toShortString(),
197+
match.justification())));
198+
if (matches.size() > DEFAULT_SUMMARY_CONSTRAINT_MATCH_LIMIT) {
199+
summary.append("%12s%s\n".formatted(" ", "..."));
200+
}
201+
});
202+
203+
return summary.toString();
204+
}
205+
206+
public boolean isSolutionInitialized() {
207+
return score().isSolutionInitialized();
208+
}
209+
210+
@Override
211+
public String toString() {
212+
return "Score analysis of score %s with %d constraints.".formatted(score, constraintMap.size());
213+
}
144214
}

‎core/src/main/java/ai/timefold/solver/core/impl/score/director/InnerScoreDirector.java

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -342,16 +342,8 @@ default ScoreAnalysis<Score_> buildScoreAnalysis(boolean analyzeConstraintMatche
342342
*/
343343
default ScoreAnalysis<Score_> buildScoreAnalysis(boolean analyzeConstraintMatches, ScoreAnalysisMode mode) {
344344
var score = calculateScore();
345-
switch (Objects.requireNonNull(mode)) {
346-
case RECOMMENDATION_API -> score = score.withInitScore(0);
347-
case DEFAULT -> {
348-
if (!score.isSolutionInitialized()) {
349-
throw new IllegalArgumentException("""
350-
Cannot analyze solution (%s) as it is not initialized (%s).
351-
Maybe run the solver first?"""
352-
.formatted(getWorkingSolution(), score));
353-
}
354-
}
345+
if (Objects.requireNonNull(mode) == ScoreAnalysisMode.RECOMMENDATION_API) {
346+
score = score.withInitScore(0);
355347
}
356348
var constraintAnalysisMap = new TreeMap<ConstraintRef, ConstraintAnalysis<Score_>>();
357349
for (var constraintMatchTotal : getConstraintMatchTotalMap().values()) {

‎core/src/test/java/ai/timefold/solver/core/api/score/analysis/ScoreAnalysisTest.java

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package ai.timefold.solver.core.api.score.analysis;
22

33
import static ai.timefold.solver.core.impl.score.director.InnerScoreDirector.getConstraintAnalysis;
4+
import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
5+
import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy;
46
import static org.assertj.core.api.SoftAssertions.assertSoftly;
57

68
import java.util.Arrays;
@@ -26,6 +28,131 @@ void empty() {
2628
softly.assertThat(diff.score()).isEqualTo(SimpleScore.of(0));
2729
softly.assertThat(diff.constraintMap()).isEmpty();
2830
});
31+
32+
var summary = scoreAnalysis.summarize();
33+
assertThat(summary)
34+
.isEqualTo("""
35+
Explanation of score (0):
36+
Constraint matches:
37+
""");
38+
}
39+
40+
@Test
41+
void summarize() {
42+
var constraintPackage = "constraintPackage";
43+
var constraintName1 = "constraint1";
44+
var constraintName2 = "constraint2";
45+
var constraintName3 = "constraint3";
46+
var constraintId1 = ConstraintRef.of(constraintPackage, constraintName1);
47+
var constraintId2 = ConstraintRef.of(constraintPackage, constraintName2);
48+
var constraintId3 = ConstraintRef.of(constraintPackage, constraintName3);
49+
50+
var constraintMatchTotal = new DefaultConstraintMatchTotal<>(constraintId1, SimpleScore.of(1));
51+
addConstraintMatch(constraintMatchTotal, SimpleScore.of(2), "A", "B", "C");
52+
addConstraintMatch(constraintMatchTotal, SimpleScore.of(4), "A", "B");
53+
addConstraintMatch(constraintMatchTotal, SimpleScore.of(6), "B", "C");
54+
addConstraintMatch(constraintMatchTotal, SimpleScore.of(7), "C");
55+
addConstraintMatch(constraintMatchTotal, SimpleScore.of(8));
56+
var constraintMatchTotal2 = new DefaultConstraintMatchTotal<>(constraintId2, SimpleScore.of(3));
57+
addConstraintMatch(constraintMatchTotal2, SimpleScore.of(3), "B", "C", "D");
58+
addConstraintMatch(constraintMatchTotal2, SimpleScore.of(6), "B", "C");
59+
addConstraintMatch(constraintMatchTotal2, SimpleScore.of(9), "C", "D");
60+
addConstraintMatch(constraintMatchTotal2, SimpleScore.of(10), "D");
61+
addConstraintMatch(constraintMatchTotal2, SimpleScore.of(12));
62+
var emptyConstraintMatchTotal1 = new DefaultConstraintMatchTotal<>(constraintId3, SimpleScore.of(0));
63+
var constraintAnalysisMap = Map.of(
64+
constraintMatchTotal.getConstraintRef(), getConstraintAnalysis(constraintMatchTotal, true),
65+
constraintMatchTotal2.getConstraintRef(), getConstraintAnalysis(constraintMatchTotal2, true),
66+
emptyConstraintMatchTotal1.getConstraintRef(), getConstraintAnalysis(emptyConstraintMatchTotal1, true));
67+
var scoreAnalysis = new ScoreAnalysis<>(SimpleScore.of(67), constraintAnalysisMap);
68+
69+
// Single constraint analysis
70+
var constraintSummary = constraintAnalysisMap.get(constraintMatchTotal.getConstraintRef()).summarize();
71+
assertThat(constraintSummary)
72+
.isEqualTo("""
73+
Explanation of score (27):
74+
Constraint matches:
75+
27: constraint (constraint1) has 5 matches:
76+
2: justified with ([A, B, C])
77+
4: justified with ([A, B])
78+
6: justified with ([B, C])
79+
...
80+
""");
81+
82+
// Complete score analysis
83+
var summary = scoreAnalysis.summarize();
84+
assertThat(scoreAnalysis.getConstraintAnalysis(constraintPackage, constraintName1).matchCount()).isEqualTo(5);
85+
assertThat(summary)
86+
.isEqualTo("""
87+
Explanation of score (67):
88+
Constraint matches:
89+
0: constraint (constraint3) has no matches.
90+
27: constraint (constraint1) has 5 matches:
91+
2: justified with ([A, B, C])
92+
4: justified with ([A, B])
93+
6: justified with ([B, C])
94+
...
95+
40: constraint (constraint2) has 5 matches:
96+
3: justified with ([B, C, D])
97+
6: justified with ([B, C])
98+
9: justified with ([C, D])
99+
...
100+
""");
101+
}
102+
103+
@Test
104+
void summarizeUninitializedSolution() {
105+
var constraintPackage = "constraintPackage";
106+
var constraintName1 = "constraint1";
107+
var constraintName2 = "constraint2";
108+
var constraintId1 = ConstraintRef.of(constraintPackage, constraintName1);
109+
var constraintId2 = ConstraintRef.of(constraintPackage, constraintName2);
110+
111+
var constraintMatchTotal = new DefaultConstraintMatchTotal<>(constraintId1, SimpleScore.of(0));
112+
var constraintMatchTotal2 = new DefaultConstraintMatchTotal<>(constraintId2, SimpleScore.of(0));
113+
var constraintAnalysisMap = Map.of(
114+
constraintMatchTotal.getConstraintRef(), getConstraintAnalysis(constraintMatchTotal, true),
115+
constraintMatchTotal2.getConstraintRef(), getConstraintAnalysis(constraintMatchTotal2, true));
116+
var scoreAnalysis = new ScoreAnalysis<>(SimpleScore.ofUninitialized(3, 0), constraintAnalysisMap);
117+
118+
// Single constraint analysis
119+
var constraintSummary = constraintAnalysisMap.get(constraintMatchTotal.getConstraintRef()).summarize();
120+
assertThat(constraintSummary)
121+
.isEqualTo("""
122+
Explanation of score (0):
123+
Constraint matches:
124+
0: constraint (constraint1) has no matches.
125+
""");
126+
127+
// Complete score analysis
128+
var summary = scoreAnalysis.summarize();
129+
assertThat(scoreAnalysis.getConstraintAnalysis(constraintPackage, constraintName1).matchCount()).isZero();
130+
assertThat(summary)
131+
.isEqualTo("""
132+
Explanation of score (3init/0):
133+
Constraint matches:
134+
0: constraint (constraint1) has no matches.
135+
0: constraint (constraint2) has no matches.
136+
""");
137+
}
138+
139+
@Test
140+
void failFastSummarize() {
141+
var constraintPackage = "constraintPackage";
142+
var constraintName1 = "constraint1";
143+
var constraintId1 = ConstraintRef.of(constraintPackage, constraintName1);
144+
145+
var constraintMatchTotal = new DefaultConstraintMatchTotal<>(constraintId1, SimpleScore.of(1));
146+
addConstraintMatch(constraintMatchTotal, SimpleScore.of(2), "A", "B", "C");
147+
var constraintAnalysisMap = Map.of(
148+
constraintMatchTotal.getConstraintRef(), getConstraintAnalysis(constraintMatchTotal, false));
149+
var scoreAnalysis = new ScoreAnalysis<>(SimpleScore.of(3), constraintAnalysisMap);
150+
151+
assertThatThrownBy(scoreAnalysis::summarize)
152+
.hasMessageContaining("The constraint matches must be non-null");
153+
154+
assertThatThrownBy(() -> constraintAnalysisMap.values().stream().findFirst().get().matchCount())
155+
.hasMessageContaining("The constraint matches must be non-null");
29156
}
30157

31158
@Test

0 commit comments

Comments
 (0)