Add support for LTWA (issue JabRef#12276) (JabRef#12880)

* Implement LTWA abbreviation

* Create LTWA resource download in the gradle tasks

* Connected the LTWA logic with the GUI

* Updated the CHANGELOG.md

* Fix according to Trag

* Reimplement with Antlr

* Fix errors from previous PR

* Use Optional as Trag suggested

* Fix the locale translation issue

---------

Co-authored-by: Christoph <[email protected]>

Author: 2 people

CHANGELOG.md

@@ -28,6 +28,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 - We added a new functionality where author names having multiple spaces in-between will be considered as separate user block as it does for " and ". [#12701](https://github.com/JabRef/jabref/issues/12701)
 - We enhanced support for parsing XMP metadata from PDF files. [#12829](https://github.com/JabRef/jabref/issues/12829)
 - We added a "Preview" header in the JStyles tab in the "Select style" dialog, to make it consistent with the CSL styles tab. [#12838](https://github.com/JabRef/jabref/pull/12838)
+- We added a "LTWA" abbreviation feature in the "Quality > Abbreviate journal names > LTWA" menu [#12273](https://github.com/JabRef/jabref/issues/12273/)
 - We added path validation to file directories in library properties dialog. [#11840](https://github.com/JabRef/jabref/issues/11840)
 
 ### Changed

build.gradle

@@ -459,14 +459,15 @@ processResources {
 tasks.register('generateSource') {
     dependsOn("generateBstGrammarSource",
             "generateSearchGrammarSource",
+            "generateLtwaGrammarSource",
             "generateCitaviSource")
     group = 'JabRef'
     description 'Generates all necessary (Java) source files.'
 }
 
 tasks.register("generateBstGrammarSource", JavaExec) {
     group = "JabRef"
-    description = 'Generates BstLexer.java and BstParser.java from the Bst.g grammar file using antlr4.'
+    description = 'Generates BstLexer.java and BstParser.java from the Bst.g4 grammar file using antlr4.'
     classpath = configurations.antlr4
     mainClass = "org.antlr.v4.Tool"
     javaLauncher.set(javaToolchains.launcherFor(java.toolchain))
@@ -478,7 +479,7 @@ tasks.register("generateBstGrammarSource", JavaExec) {
 
 tasks.register("generateSearchGrammarSource", JavaExec) {
     group = 'JabRef'
-    description = "Generates java files for Search.g antlr4."
+    description = "Generates java files for Search.g4 antlr4."
     classpath = configurations.antlr4
     mainClass = "org.antlr.v4.Tool"
     javaLauncher.set(javaToolchains.launcherFor(java.toolchain))
@@ -488,6 +489,18 @@ tasks.register("generateSearchGrammarSource", JavaExec) {
     args  = ["-o","src-gen/main/java/org/jabref/search" , "-visitor", "-no-listener", "-package", "org.jabref.search", "$projectDir/src/main/antlr4/org/jabref/search/Search.g4"]
 }
 
+tasks.register("generateLtwaGrammarSource", JavaExec) {
+    group = "JabRef"
+    description = 'Generates LtwaLexer.java and LtwaParser.java from the Ltwa.g4 grammar file using antlr4.'
+    classpath = configurations.antlr4
+    mainClass = "org.antlr.v4.Tool"
+    javaLauncher.set(javaToolchains.launcherFor(java.toolchain))
+
+    inputs.dir('src/main/antlr4/org/jabref/ltwa/')
+    outputs.dir("src-gen/main/java/org/jabref/logic/journals/ltwa/")
+    args = ["-o", "src-gen/main/java/org/jabref/logic/journals/ltwa/", "-no-visitor", "-listener", "-package", "org.jabref.logic.journals.ltwa", "$projectDir/src/main/antlr4/org/jabref/ltwa/Ltwa.g4"]
+}
+
 tasks.register("generateJournalListMV", JavaExec) {
     group = "JabRef"
     description = "Converts the comma-separated journal abbreviation file to a H2 MVStore"
@@ -502,6 +515,42 @@ tasks.register("generateJournalListMV", JavaExec) {
 jar.dependsOn("generateJournalListMV")
 compileTestJava.dependsOn("generateJournalListMV")
 
+tasks.register('downloadLtwaFile') {
+    group = "JabRef"
+    description = "Downloads the LTWA file for journal abbreviations"
+    doLast {
+        def ltwaUrl = "https://www.issn.org/wp-content/uploads/2021/07/ltwa_20210702.csv"
+        def ltwaDir = file("build/resources/main/journals")
+        def ltwaCsvFile = file("${ltwaDir}/ltwa_20210702.csv")
+
+        if (!ltwaCsvFile.exists()) {
+            mkdir(ltwaDir)
+            ant.get(src: ltwaUrl, dest: ltwaCsvFile, verbose: true)
+            logger.lifecycle("Downloaded LTWA file to ${ltwaCsvFile}")
+        } else {
+            logger.lifecycle("LTWA file already exists at ${ltwaCsvFile}")
+        }
+    }
+    onlyIf {
+        !file("build/resources/main/journals/ltwa_20210702.csv").exists()
+    }
+}
+
+tasks.register('generateLtwaListMV', JavaExec) {
+    group = "JabRef"
+    description = "Converts the LTWA CSV file to a H2 MVStore"
+    classpath = sourceSets.main.runtimeClasspath
+    mainClass = "org.jabref.cli.LtwaListMvGenerator"
+    javaLauncher.set(javaToolchains.launcherFor(java.toolchain))
+    dependsOn('downloadLtwaFile')
+    onlyIf {
+        !file("build/resources/main/journals/ltwa-list.mv").exists()
+    }
+}
+
+jar.dependsOn("generateLtwaListMV")
+compileTestJava.dependsOn("generateLtwaListMV")
+
 tasks.register('generateCitaviSource', XjcTask) {
     group = 'JabRef'
     description = "Generates java files for the citavi importer."

src/main/antlr4/org/jabref/ltwa/Ltwa.g4

@@ -0,0 +1,110 @@
+grammar Ltwa;
+options { caseInsensitive=true; }
+
+@lexer::members {
+    private boolean isNextBoundary() {
+        int lookAhead = _input.LA(1);
+        return lookAhead == EOF ||
+               lookAhead == ' ' || lookAhead == '\t' || lookAhead == '\r' || lookAhead == '\n' ||
+               lookAhead == '-' || lookAhead == '\u2013' || lookAhead == '\u2014' ||
+               lookAhead == '_' || lookAhead == '.' || lookAhead == ',' ||
+               lookAhead == ':' || lookAhead == ';' || lookAhead == '!' ||
+               lookAhead == '|' || lookAhead == '=' || lookAhead == '+' ||
+               lookAhead == '*' || lookAhead == '\\' || lookAhead == '/' ||
+               lookAhead == '"' || lookAhead == '(' || lookAhead == ')' ||
+               lookAhead == '&' || lookAhead == '#' || lookAhead == '%' ||
+               lookAhead == '@' || lookAhead == '$' || lookAhead == '?';
+    }
+
+    private boolean isNotHyphenated() {
+        int lookAhead = _input.LA(1);
+        return lookAhead == EOF || lookAhead != '-';
+    }
+}
+
+fragment COMMON_ABBR_FRAGMENT options { caseInsensitive=false; }: 'St' | 'Mr' | 'Ms' | 'Mrs' | 'Mx' | 'Dr' | 'Prof' | 'vs';
+fragment PART_ABBR_FRAGMENT: 'ser' | 'sect' | 'sec';
+
+ABBREVIATION
+    : (COMMON_ABBR_FRAGMENT | PART_ABBR_FRAGMENT) '.' {isNextBoundary()}?
+    | ([A-Z] '.')+ {isNextBoundary()}?;
+
+PART: ('series' | 'serie' | 'part' | 'section' | 'série' | 'supplemento' | 'chapter' | 'parte') {isNextBoundary()}?;
+
+ORDINAL options { caseInsensitive=false; }: ([IVXivx]+ | [A-Z]) {isNextBoundary()}?;
+
+ARTICLE: ('l\'' | 'd\'' | 'dell\'' | 'nell\'')
+       | ('a' | 'an' | 'the'
+       | 'der' | 'die' | 'das' | 'des' | 'dem' | 'den'
+       | 'el' | 'la' | 'los' | 'las' | 'un' | 'una' | 'unos' | 'unas'
+       | 'le' | 'la' | 'les' | 'un' | 'une' | 'des' | 'du' | 'de la' | 'au' | 'aux'
+       | 'dell' | 'nell') {isNextBoundary()}? {isNotHyphenated()}?;
+
+STOPWORD: ('a' | 'an' | 'the' | 'and' | 'but' | 'or' | 'for' | 'nor' | 'so' | 'yet' | 'though'
+        | 'when' | 'whenever' | 'where' | 'whereas' | 'wherever' | 'while' | 'about' | 'afore'
+        | 'after' | 'ago' | 'along' | 'amid' | 'among' | 'amongst' | 'apropos' | 'as' | 'at'
+        | 'atop' | 'by' | 'ca' | 'circa' | 'from' | 'hence' | 'in' | 'into'
+        | 'like' | 'of' | 'off' | 'on' | 'onto' | 'ontop' | 'out' | 'over' | 'per' | 'since'
+        | 'than' | 'til' | 'till' | 'to' | 'unlike' | 'until' | 'unto' | 'up' | 'upon' | 'upside'
+        | 'versus' | 'via' | 'vis-a-vis' | 'vs' | 'with' | 'within' | 'für' | 'und' | 'aus'
+        | 'zu' | 'zur' | 'im' | 'de' | 'et' | 'y' | 'del' | 'en' | 'di' | 'e' | 'da' | 'delle'
+        | 'della' | 'sue' | 'el' | 'do' | 'og' | 'i' | 'voor' | 'van' | 'dell\'' | 'dell' | 'ed'
+        | 'för' | 'tot' | 'vir' | 'o' | 'its' | 'sul') {isNextBoundary()}? {isNotHyphenated()}?;
+
+HYPHEN: '-';
+
+SYMBOLS: [.,;!?&+=*#%@$] | '\'';
+
+fragment LETTER: 'A'..'Z' | 'À'..'Ö' | 'ø'..'ÿ' | '\u0100'..'\u017F' | '\u4E00'..'\u9FFF';
+
+WORD: (LETTER+ '\'' + [a-z]) {isNextBoundary()}?              // e.g., Shi'a, parent's
+    | (LETTER+ '\'') {isNextBoundary()}?                      // Word ending with apostrophe, e.g., Parents' (plural possessive)
+    | LETTER + ('.' + LETTER+)+ {isNextBoundary()}?           // e.g., Humana.Mente
+    | (LETTER | [0-9])+ {isNextBoundary()}?;                  // Regular word
+
+WS: [ \t\r\n]+ -> skip;
+
+// Parser rules
+title
+    : singleWordTitle EOF  #SingleWordTitleFull
+    | stopwordPlusAny EOF #StopwordPlusTitleFull
+    | anyPlusSymbols EOF  #AnyPlusSymbolsFull
+    | normalTitle EOF     #NormalTitleFull
+    ;
+
+singleWordTitle
+    : (WORD | STOPWORD | PART | ORDINAL | ABBREVIATION)
+    ;
+
+stopwordPlusAny
+    : STOPWORD (WORD | PART | ORDINAL | ABBREVIATION)
+    ;
+
+anyPlusSymbols
+    : (WORD | STOPWORD | PART | ORDINAL | ABBREVIATION) SYMBOLS
+    ;
+
+normalTitle
+    : titleElement+
+    ;
+
+titleElement
+    : article      #ArticleElement
+    | stopword     #StopwordElement
+    | symbols      #SymbolsElement
+    | ordinal      #OrdinalElement
+    | word         #WordElement
+    | hyphen       #HyphenElement
+    | part         #PartElement
+    | abbreviation #AbbreviationElement
+    ;
+
+// Rules for each token type
+article      : ARTICLE;
+stopword     : STOPWORD;
+symbols      : SYMBOLS;
+ordinal      : ORDINAL;
+word         : WORD;
+hyphen       : HYPHEN;
+part         : PART;
+abbreviation : ABBREVIATION;

src/main/java/org/jabref/cli/LtwaListMvGenerator.java

@@ -0,0 +1,113 @@
+package org.jabref.cli;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.jabref.logic.journals.ltwa.LtwaEntry;
+import org.jabref.logic.journals.ltwa.LtwaTsvParser;
+import org.jabref.logic.journals.ltwa.NormalizeUtils;
+import org.jabref.logic.journals.ltwa.PrefixTree;
+
+import org.h2.mvstore.MVMap;
+import org.h2.mvstore.MVStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * CLI tool for downloading the LTWA CSV file and converting it to an MVStore file.
+ */
+public class LtwaListMvGenerator {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(LtwaListMvGenerator.class);
+    private static final String LTWA_URL = "https://www.issn.org/wp-content/uploads/2021/07/ltwa_20210702.csv";
+
+    public static void main(String[] args) {
+        try {
+            Path tempCsvFile = downloadLtwaFile();
+            Path outputDir = Path.of("build", "resources", "main", "journals");
+            Files.createDirectories(outputDir);
+            Path outputFile = outputDir.resolve("ltwa-list.mv");
+
+            generateMvStore(tempCsvFile, outputFile);
+
+            // Delete temp file
+            Files.deleteIfExists(tempCsvFile);
+
+            LOGGER.info("LTWA MVStore file generated successfully at {}.", outputFile);
+        } catch (IOException e) {
+            LOGGER.error("Error generating LTWA MVStore file.", e);
+        } catch (URISyntaxException e) {
+            LOGGER.error("Invalid URL for LTWA file (this should never happen).", e);
+        }
+    }
+
+    /**
+     * Downloads the LTWA CSV file from the specified URL.
+     *
+     * @return Path to the downloaded file
+     * @throws IOException If an I/O error occurs
+     */
+    private static Path downloadLtwaFile() throws IOException, URISyntaxException {
+        LOGGER.info("Downloading LTWA file from {}.", LtwaListMvGenerator.LTWA_URL);
+        var in = new URI(LTWA_URL).toURL().openStream();
+        var path = Files.writeString(
+                Files.createTempFile("ltwa", ".csv"),
+                new String(in.readAllBytes()),
+                StandardOpenOption.CREATE,
+                StandardOpenOption.TRUNCATE_EXISTING);
+        in.close();
+        return path;
+    }
+
+    /**
+     * Generates an MVStore file from the LTWA CSV file.
+     *
+     * @param inputFile  Path to the LTWA CSV file
+     * @param outputFile Path where the MVStore file will be written
+     * @throws IOException If an I/O error occurs
+     */
+    private static void generateMvStore(Path inputFile, Path outputFile) throws IOException {
+        LOGGER.info("Parsing LTWA file...");
+        LtwaTsvParser parser = new LtwaTsvParser(inputFile);
+        List<LtwaEntry> entries = parser.parse();
+
+        LOGGER.info("Found {} LTWA entries", entries.size());
+
+        try (MVStore store = new MVStore.Builder()
+                .fileName(outputFile.toString())
+                .compressHigh()
+                .open()) {
+            MVMap<String, List<LtwaEntry>> prefixMap = store.openMap("Prefixes");
+            MVMap<String, List<LtwaEntry>> suffixMap = store.openMap("Suffixes");
+            String inflection = Character.toString(PrefixTree.WILD_CARD).repeat(3) + " ";
+
+            entries.forEach(entry ->
+                    NormalizeUtils.normalize(entry.word())
+                                  .map(String::toLowerCase)
+                                  .map(word -> word.replace(" ", inflection))
+                                  .ifPresent(word -> {
+                                      if (word.startsWith("-")) {
+                                          String key = word.substring(1);
+                                          suffixMap.computeIfAbsent(key, k ->
+                                                  Stream.<LtwaEntry>builder().build().collect(Collectors.toList())
+                                          ).add(entry);
+                                      } else {
+                                          String key = word.endsWith("-") ? word.substring(0, word.length() - 1) : word;
+                                          prefixMap.computeIfAbsent(key, k ->
+                                                  Stream.<LtwaEntry>builder().build().collect(Collectors.toList())
+                                          ).add(entry);
+                                      }
+                                  })
+            );
+
+            LOGGER.info("Stored {} prefixes and {} suffixes", prefixMap.size(), suffixMap.size());
+        }
+    }
+}

src/main/java/org/jabref/gui/actions/StandardActions.java

@@ -113,6 +113,7 @@ public enum StandardActions implements Action {
     ABBREVIATE_DEFAULT(Localization.lang("default"), Localization.lang("Abbreviate journal names of the selected entries (DEFAULT abbreviation)"), KeyBinding.ABBREVIATE),
     ABBREVIATE_DOTLESS(Localization.lang("dotless"), Localization.lang("Abbreviate journal names of the selected entries (DOTLESS abbreviation)")),
     ABBREVIATE_SHORTEST_UNIQUE(Localization.lang("shortest unique"), Localization.lang("Abbreviate journal names of the selected entries (SHORTEST UNIQUE abbreviation)")),
+    ABBREVIATE_LTWA(Localization.lang("LTWA"), Localization.lang("Abbreviate journal names of the selected entries (LTWA)")),
     UNABBREVIATE(Localization.lang("Unabbreviate journal names"), Localization.lang("Unabbreviate journal names of the selected entries"), KeyBinding.UNABBREVIATE),
 
     MANAGE_CUSTOM_EXPORTS(Localization.lang("Manage custom exports")),

src/main/java/org/jabref/gui/frame/MainMenu.java

@@ -265,7 +265,8 @@ private void createMenu() {
                 factory.createSubMenu(StandardActions.ABBREVIATE,
                         factory.createMenuItem(StandardActions.ABBREVIATE_DEFAULT, new AbbreviateAction(StandardActions.ABBREVIATE_DEFAULT, frame::getCurrentLibraryTab, dialogService, stateManager, preferences.getJournalAbbreviationPreferences(), abbreviationRepository, taskExecutor, undoManager)),
                         factory.createMenuItem(StandardActions.ABBREVIATE_DOTLESS, new AbbreviateAction(StandardActions.ABBREVIATE_DOTLESS, frame::getCurrentLibraryTab, dialogService, stateManager, preferences.getJournalAbbreviationPreferences(), abbreviationRepository, taskExecutor, undoManager)),
-                        factory.createMenuItem(StandardActions.ABBREVIATE_SHORTEST_UNIQUE, new AbbreviateAction(StandardActions.ABBREVIATE_SHORTEST_UNIQUE, frame::getCurrentLibraryTab, dialogService, stateManager, preferences.getJournalAbbreviationPreferences(), abbreviationRepository, taskExecutor, undoManager))),
+                        factory.createMenuItem(StandardActions.ABBREVIATE_SHORTEST_UNIQUE, new AbbreviateAction(StandardActions.ABBREVIATE_SHORTEST_UNIQUE, frame::getCurrentLibraryTab, dialogService, stateManager, preferences.getJournalAbbreviationPreferences(), abbreviationRepository, taskExecutor, undoManager)),
+                        factory.createMenuItem(StandardActions.ABBREVIATE_LTWA, new AbbreviateAction(StandardActions.ABBREVIATE_LTWA, frame::getCurrentLibraryTab, dialogService, stateManager, preferences.getJournalAbbreviationPreferences(), abbreviationRepository, taskExecutor, undoManager))),
 
                 factory.createMenuItem(StandardActions.UNABBREVIATE, new AbbreviateAction(StandardActions.UNABBREVIATE, frame::getCurrentLibraryTab, dialogService, stateManager, preferences.getJournalAbbreviationPreferences(), abbreviationRepository, taskExecutor, undoManager))
         );

src/main/java/org/jabref/gui/journals/AbbreviateAction.java

@@ -60,10 +60,16 @@ public AbbreviateAction(StandardActions action,
         this.undoManager = undoManager;
 
         switch (action) {
-            case ABBREVIATE_DEFAULT -> abbreviationType = AbbreviationType.DEFAULT;
-            case ABBREVIATE_DOTLESS -> abbreviationType = AbbreviationType.DOTLESS;
-            case ABBREVIATE_SHORTEST_UNIQUE -> abbreviationType = AbbreviationType.SHORTEST_UNIQUE;
-            default -> LOGGER.debug("Unknown action: {}", action.name());
+            case ABBREVIATE_DEFAULT ->
+                    abbreviationType = AbbreviationType.DEFAULT;
+            case ABBREVIATE_DOTLESS ->
+                    abbreviationType = AbbreviationType.DOTLESS;
+            case ABBREVIATE_SHORTEST_UNIQUE ->
+                    abbreviationType = AbbreviationType.SHORTEST_UNIQUE;
+            case ABBREVIATE_LTWA ->
+                    abbreviationType = AbbreviationType.LTWA;
+            default ->
+                    LOGGER.debug("Unknown action: {}", action.name());
         }
 
         this.executable.bind(ActionHelper.needsEntriesSelected(stateManager));
@@ -73,7 +79,8 @@ public AbbreviateAction(StandardActions action,
     public void execute() {
         if ((action == StandardActions.ABBREVIATE_DEFAULT)
                 || (action == StandardActions.ABBREVIATE_DOTLESS)
-                || (action == StandardActions.ABBREVIATE_SHORTEST_UNIQUE)) {
+                || (action == StandardActions.ABBREVIATE_SHORTEST_UNIQUE)
+                || (action == StandardActions.ABBREVIATE_LTWA)) {
             dialogService.notify(Localization.lang("Abbreviating..."));
             stateManager.getActiveDatabase().ifPresent(_ ->
                     BackgroundTask.wrap(() -> abbreviate(stateManager.getActiveDatabase().get(), stateManager.getSelectedEntries()))

src/main/java/org/jabref/gui/journals/AbbreviationType.java

@@ -2,9 +2,15 @@
 
 /**
  * Defines the different abbreviation types that JabRef can operate with.
+ *
+ * DEFAULT: Default abbreviation type, which is the standard behavior.
+ * DOTLESS: Abbreviation type that does not include dots in the abbreviation.
+ * SHORTEST_UNIQUE: Abbreviation type that generates the shortest unique abbreviation.
+ * LTWA: Abbreviation type that uses the LTWA (List of Title Word Abbreviations)/ISO4 method.
  */
 public enum AbbreviationType {
     DEFAULT,
     DOTLESS,
-    SHORTEST_UNIQUE
+    SHORTEST_UNIQUE,
+    LTWA
 }