diff --git a/README.md b/README.md index 6721448cf..7c807077e 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,8 @@ An application that identifies the part of speech (e.g. verb + tense, noun + num in plain text. - [illinois-ner](ner/README.md) An application that identifies named entities in plain text according to two different sets of categories. - + - [illinois-srl](srl/README.md) +An application to annotate natural language sentences with semantic roles. ## Using each library programmatically diff --git a/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationCache.java b/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationCache.java index d12384fa6..e756735d7 100644 --- a/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationCache.java +++ b/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationCache.java @@ -26,4 +26,6 @@ public interface TextAnnotationCache { boolean contains(TextAnnotation ta); void removeTextAnnotation(TextAnnotation ta); + + void close(); } diff --git a/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationDBHandler.java b/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationDBHandler.java index 321b9b90e..6a94fa4a8 100644 --- a/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationDBHandler.java +++ b/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationDBHandler.java @@ -276,4 +276,7 @@ public void removeTextAnnotation(TextAnnotation ta) { throw new RuntimeException(e); } } + + @Override + public void close() {} } diff --git a/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationMapDBHandler.java b/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationMapDBHandler.java index 27cf28bcc..b5b6833fd 100644 --- a/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationMapDBHandler.java +++ b/core-utilities/src/main/java/edu/illinois/cs/cogcomp/core/io/caches/TextAnnotationMapDBHandler.java @@ -11,7 +11,6 @@ import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; import edu.illinois.cs.cogcomp.core.io.IOUtils; import edu.illinois.cs.cogcomp.core.utilities.SerializationHelper; -import org.jetbrains.annotations.NotNull; import org.mapdb.DB; import org.mapdb.DBMaker; import org.mapdb.Serializer; @@ -127,7 +126,6 @@ private ConcurrentMap getMap(String dataset) { } @SuppressWarnings("ConstantConditions") - @NotNull private Iterable getAllDatasets() { ReentrantReadWriteLock.ReadLock lock = db.getLock$mapdb().readLock(); lock.tryLock(); diff --git a/pom.xml b/pom.xml index f47755d55..b2351296a 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,7 @@ pos chunker ner + srl corpusreaders lbjava-nlp-tools big-data-utils diff --git a/srl/CHANGELOG b/srl/CHANGELOG new file mode 100644 index 000000000..be59c33a2 --- /dev/null +++ b/srl/CHANGELOG @@ -0,0 +1,55 @@ +Version 3.0.73 +Moved to the super-project and changed the versioning to the super-project versioning + +Version 5.1.12 +Added Windows support (including access to non-Gurobi solver) + +Version 5.1.4 +Switched entirely to illinois-sl for structured prediction (removed JLIS traces) +Using the latest AnnotatorService from illinois-core-utilities for both Curator & pipeline annotation +Major cleaning up + +Version 5.1 +Added JUnit tests +Removed unnecessary dependencies +Switched to illinois-nlp-pipeline-0.1.2 +Minor fixes + +Version 5.0 +Standalone SRL using illinois-nlp-pipeline + +Version 4.1.1 +Switched to edison-0.7.1 and LBJava-1.0 +Added dependency to illinois-common-resources + +Version 4.1 +Various bugfixes + +Version 4.0.2 +Updated inference dependency to latest version and modified inference +code accordingly. + +Version 4.0.1 +Removed duplicate code from JLIS-core and moved to IllinoisSL. Minor edits. + +Version 4.0 +A complete rewrite of the SRL. Includes predicate and sense detectors, +new constraints and a memory footprint of only 3GB. + +Version 3.0.3 +Minor bugfixes. Uses edison v0.2.9 + +Version 3.0.2 +Added an option to trim leading prepositions from arguments. + +Revamped the training mechanism to train using LBJ's BatchTrainer in +the code. This allows manual lexicon handling, which reduces the +memory requirements by nearly 40 percent. + +Version 3.0.1 +Minor bugfix + +Version 3.0 +A complete Java based re-implementation of the Illinois SRL from +Punyakanok 2008. This version uses LBJ to train classifiers and +for performing inference with a home-brewed beam search. diff --git a/srl/README.md b/srl/README.md new file mode 100644 index 000000000..334459eaf --- /dev/null +++ b/srl/README.md @@ -0,0 +1,49 @@ +# illinois-srl: Semantic Role Labeler + +### Running +You can use the **illinois-srl** system in either *interactive* or *annotator* mode. +#### Interactive mode +In *interactive mode* the user can input a single piece of text and get back the feedback from both +the **Nom**inal or **Verb**al SRL systems in plain text. + +To run the system in *interactive mode* see the class `edu.illinois.cs.cogcomp.srl.SemanticRoleLabeler` +or simply execute the `run-interactive` script: + +For linux: +``` +scripts/run-interactive.sh +``` + +For windows: +``` +cd scripts +run-interactive-win.bat +``` + +#### As an `Annotator` component +**illinois-srl** can also be used programmatically through the `SemanticRoleLabeler` class which implements CogComp's +[Annotator interface](http://cogcomp.cs.illinois.edu/software/doc/illinois-core-utilities/apidocs/edu/illinois/cs/cogcomp/core/datastructures/textannotation/Annotator.html). + +The main method is `getView(TextAnnotation)` inside `SemanticRoleLabeler`. This will add a new +[`PredicateArgumentView`](http://cogcomp.cs.illinois.edu/software/doc/illinois-core-utilities/apidocs/edu/illinois/cs/cogcomp/core/datastructures/textannotation/PredicateArgumentView.html) +for either **Nom**inal or **Verb**al SRL. + +### Training +To train the SRL system you will require access to the [Propbank](https://verbs.colorado.edu/~mpalmer/projects/ace.html) +or [Nombank](http://nlp.cs.nyu.edu/meyers/NomBank.html) corpora. You need to set pointers to these in the +`config/srl-config.properties` file. +(To train the system with a non-Prop/Nombank corpus, you need to extend +[`AbstractSRLAnnotationReader`](http://cogcomp.cs.illinois.edu/software/doc/illinois-core-utilities/apidocs/edu/illinois/cs/cogcomp/nlp/corpusreaders/AbstractSRLAnnotationReader.html)) + +To perform the whole training/testing suite, run the `Main` class with parameters ` expt Verb|Nom true`. +This will: + +1. Read and cache the datasets (train/test) +2. Annotate each `TextAnnotation` with the required views + (here you can set the `useCurator` flag to false to use the CogComp's standalone NLP pipeline) +3. Pre-extract and cache the features for the classifiers +4. Train the classifiers +5. Evaluate on the (cached) test corpus + +**IMPORTANT** After training, make sure you comment-out the pre-trained SRL model dependencies inside +`pom.xml` (lines 27-38). \ No newline at end of file diff --git a/srl/config/learner.properties b/srl/config/learner.properties new file mode 100644 index 000000000..ac9f22933 --- /dev/null +++ b/srl/config/learner.properties @@ -0,0 +1,20 @@ +# Available learning models: {L2LossSSVM, StructuredPerceptron} +LEARNING_MODEL = L2LossSSVM + +# Available solver types: {DCDSolver, ParallelDCDSolver, DEMIParallelDCDSolver} +L2_LOSS_SSVM_SOLVER_TYPE = ParallelDCDSolver + +NUMBER_OF_THREADS = 8 + +# Regularization parameter +C_FOR_STRUCTURE = 1.0 + +# Mini-batch for 'warm' start +TRAINMINI = true +TRAINMINI_SIZE = 10000 + +# Suppress optimatility check +CHECK_INFERENCE_OPT = false + +# Number of training rounds +MAX_NUM_ITER = 100 diff --git a/srl/config/pipeline.properties b/srl/config/pipeline.properties new file mode 100644 index 000000000..ca6c09da6 --- /dev/null +++ b/srl/config/pipeline.properties @@ -0,0 +1,17 @@ +## Flags for whether to use different annotators +usePos = true +useLemma = true +useShallowParse = true +useNerConll = true +useNerOntonotes = false +useStanfordParse = true +useStanfordDep = true +useSrlVerb = false +useSrlNom = false + +## Flags for the Stanford parser (for pre-processing) +# Max time per sentence (in milliseconds) +stanfordMaxTimePerSentence = 1000 + +# Max sentence lenght (will throw exception if larger) +stanfordParseMaxSentenceLength = 80 \ No newline at end of file diff --git a/srl/config/srl-config.properties b/srl/config/srl-config.properties new file mode 100644 index 000000000..356119b02 --- /dev/null +++ b/srl/config/srl-config.properties @@ -0,0 +1,43 @@ +## Illinois SRL Configuration## + +# Whether to use the Illinois Curator to get the required annotations for training/testing +# If set to false, Illinois NLP pipeline will be used +UseCurator = false + +# The configuration of the Illinois NLP pipeline +PipelineConfig = config/pipeline.properties + +# The parser used to extract constituents and syntactic features +# Options are: Charniak, Berkeley, Stanford +# NB: Only Stanford can be used in standalone mode. +DefaultParser = Stanford + +# The configuration for the Structured learner +LearnerConfig = config/learner.properties + +# Num of threads for feat. ext. +NumFeatExtThreads = 10 + +# The ILP solver to use for the joint inference +# Options are: Gurobi, OJAlgo +ILPSolver = OJAlgo + +# The TextAnnotation caching mechanism to use +# Options are: MapDB, H2 +DatasetCache = MapDB + +### Training corpora directories ### +# This is the directory of the merged (mrg) WSJ files +PennTreebankHome = /shared/corpora/corporaWeb/treebanks/eng/pennTreebank/treebank-3/parsed/mrg/wsj/ +PropbankHome = /shared/corpora/corporaWeb/treebanks/eng/propbank_1/data +NombankHome = /shared/corpora/corporaWeb/treebanks/eng/nombank/ + +# The directory of the sentence and pre-extracted features database (~5G of space required) +# Not used during test/working with pre-trained models +CacheDirectory = cache + +ModelsDirectory = models + +# Directory to output gold and predicted files for manual comparison +# Comment out for no output +OutputDirectory = srl-out \ No newline at end of file diff --git a/srl/pom.xml b/srl/pom.xml new file mode 100644 index 000000000..bb85f36b8 --- /dev/null +++ b/srl/pom.xml @@ -0,0 +1,133 @@ + + + + illinois-cogcomp-nlp + edu.illinois.cs.cogcomp + 3.0.77 + + + 4.0.0 + illinois-srl + jar + http://cogcomp.cs.illinois.edu + + + UTF-8 + UTF-8 + 0.1.24 + + + + + + + edu.illinois.cs.cogcomp + illinois-srl-models + verb-stanford + 5.1 + + + edu.illinois.cs.cogcomp + illinois-srl-models + nom-stanford + 5.1 + + + + + edu.illinois.cs.cogcomp + illinois-nlp-pipeline + ${cogcomp-nlp-pipeline-version} + + + illinois-srl + edu.illinois.cs.cogcomp + + + + + + + edu.illinois.cs.cogcomp + illinois-core-utilities + 3.0.77 + + + edu.illinois.cs.cogcomp + illinois-curator + 3.0.77 + + + edu.illinois.cs.cogcomp + illinois-edison + 3.0.77 + + + com.gurobi + gurobi + 6.5 + true + + + edu.illinois.cs.cogcomp + illinois-common-resources + illinoisSRL + 1.5 + + + edu.illinois.cs.cogcomp + illinois-common-resources + 1.5 + + + edu.illinois.cs.cogcomp + illinois-common-resources + ner + 1.5 + + + edu.illinois.cs.cogcomp + illinois-sl-core + 1.0.3 + + + commons-lang + commons-lang + 2.6 + + + com.h2database + h2 + 1.4.190 + + + edu.illinois.cs.cogcomp + illinois-inference + 0.6.0 + + + org.tartarus + snowball + 1.0 + + + junit + junit + 4.12 + test + + + + + true + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.3 + + + + + diff --git a/srl/scripts/deployModels.sh b/srl/scripts/deployModels.sh new file mode 100755 index 000000000..a02eb7986 --- /dev/null +++ b/srl/scripts/deployModels.sh @@ -0,0 +1,100 @@ +#!/bin/sh + +## Important - change this ONLY when the minor version changes +VERSION=5.1 + +tmpdir=tmp-Srl-verb-$RANDOM + +rm -rdf ${tmpdir} +mkdir -p ${tmpdir}/models + +for parser in STANFORD CHARNIAK; do + if [ ! -e "./models/Verb.Classifier.PARSE_${parser}.lex" ]; then + echo "$parser Verb models not found" + continue + fi + + cp ./models/Verb*${parser}* ${tmpdir}/models + + cd ${tmpdir} + rm -rdf ../target/illinois-srl-models-verb-${parser}-${VERSION}.jar + jar cf ../target/illinois-srl-models-verb-${parser}-${VERSION}.jar models + cd .. + rm ${tmpdir}/models/* +done +rm -rdf ${tmpdir} + +tmpdir=tmp-Srl-nom-$RANDOM + +rm -rdf ${tmpdir} +mkdir -p ${tmpdir}/models + +for parser in STANFORD CHARNIAK; do + if [ ! -e "./models/Nom.Classifier.PARSE_${parser}.lex" ]; then + echo "$parser Nom models not found" + continue + fi + + cp ./models/Nom*${parser}* ${tmpdir}/models + + cd ${tmpdir} + rm -rdf ../target/illinois-srl-models-nom-${parser}-${VERSION}.jar + jar cf ../target/illinois-srl-models-nom-${parser}-${VERSION}.jar models + cd .. + rm ${tmpdir}/models/* +done +rm -rdf ${tmpdir} + +echo "Compiled models to jars" + +if [ -e "target/illinois-srl-models-nom-CHARNIAK-${VERSION}.jar" ]; then +echo "Deploying illinois-srl-models-nom-CHARNIAK-${VERSION}.jar" +mvn deploy:deploy-file \ + -Dfile=target/illinois-srl-models-nom-CHARNIAK-${VERSION}.jar \ + -DgroupId=edu.illinois.cs.cogcomp \ + -DartifactId=illinois-srl-models \ + -Dversion=${VERSION} \ + -Dclassifier=nom-charniak \ + -Dpackaging=jar \ + -Durl=scp://bilbo.cs.illinois.edu:/mounts/bilbo/disks/0/www/cogcomp/html/m2repo \ + -DrepositoryId=CogcompSoftware +fi + +if [ -e "target/illinois-srl-models-nom-STANFORD-${VERSION}.jar" ]; then +echo "Deploying illinois-srl-models-nom-STANFORD-${VERSION}.jar" +mvn deploy:deploy-file \ + -Dfile=target/illinois-srl-models-nom-STANFORD-${VERSION}.jar \ + -DgroupId=edu.illinois.cs.cogcomp \ + -DartifactId=illinois-srl-models \ + -Dversion=${VERSION} \ + -Dclassifier=nom-stanford \ + -Dpackaging=jar \ + -Durl=scp://bilbo.cs.illinois.edu:/mounts/bilbo/disks/0/www/cogcomp/html/m2repo \ + -DrepositoryId=CogcompSoftware +fi + +if [ -e "target/illinois-srl-models-verb-CHARNIAK-${VERSION}.jar" ]; then +echo "Deploying illinois-srl-models-verb-CHARNIAK-${VERSION}.jar" +mvn deploy:deploy-file \ + -Dfile=target/illinois-srl-models-verb-CHARNIAK-${VERSION}.jar \ + -DgroupId=edu.illinois.cs.cogcomp \ + -DartifactId=illinois-srl-models \ + -Dversion=${VERSION} \ + -Dclassifier=verb-charniak \ + -Dpackaging=jar \ + -Durl=scp://bilbo.cs.illinois.edu:/mounts/bilbo/disks/0/www/cogcomp/html/m2repo \ + -DrepositoryId=CogcompSoftware +fi + +if [ -e "target/illinois-srl-models-verb-STANFORD-${VERSION}.jar" ]; then +echo "Deploying illinois-srl-models-verb-STANFORD-${VERSION}.jar" +mvn deploy:deploy-file \ + -Dfile=target/illinois-srl-models-verb-STANFORD-${VERSION}.jar \ + -DgroupId=edu.illinois.cs.cogcomp \ + -DartifactId=illinois-srl-models \ + -Dversion=${VERSION} \ + -Dclassifier=verb-stanford \ + -Dpackaging=jar \ + -Durl=scp://bilbo.cs.illinois.edu:/mounts/bilbo/disks/0/www/cogcomp/html/m2repo \ + -DrepositoryId=CogcompSoftware +fi \ No newline at end of file diff --git a/srl/scripts/run-interactive-win.bat b/srl/scripts/run-interactive-win.bat new file mode 100644 index 000000000..371811233 --- /dev/null +++ b/srl/scripts/run-interactive-win.bat @@ -0,0 +1,19 @@ +@REM +@REM This software is released under the University of Illinois/Research and Academic Use License. See +@REM the LICENSE file in the root folder for details. Copyright (c) 2016 +@REM +@REM Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign +@REM http://cogcomp.cs.illinois.edu/ +@REM + +rem Moving to root directory +cd .. + +rem Compiling the source code +call mvn clean compile + +rem Copying dependencies +call mvn -q dependency:copy-dependencies + +rem Starting the SRL system +java -Xmx8g -cp target\classes;target\dependency\* edu.illinois.cs.cogcomp.srl.SemanticRoleLabeler config\srl-config.properties \ No newline at end of file diff --git a/srl/scripts/run-interactive.sh b/srl/scripts/run-interactive.sh new file mode 100755 index 000000000..00198c8db --- /dev/null +++ b/srl/scripts/run-interactive.sh @@ -0,0 +1,13 @@ +#!/bin/bash +mvn clean compile +mvn -q dependency:copy-dependencies + +CP=target/classes:config:target/dependency/* + +MEMORY="-Xmx8g" + +OPTIONS="-ea $MEMORY -cp $CP " + +MAINCLASS=edu.illinois.cs.cogcomp.srl.SemanticRoleLabeler + +java $OPTIONS $MAINCLASS config/srl-config.properties diff --git a/srl/scripts/run.sh b/srl/scripts/run.sh new file mode 100755 index 000000000..2093283d9 --- /dev/null +++ b/srl/scripts/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash +mvn clean compile +mvn -q dependency:copy-dependencies + +CP=target/classes:config:target/dependency/* + +MEMORY="-Xmx100g" + +#OPTIONS="-ea $MEMORY -cp $CP " +OPTIONS="$MEMORY -cp $CP " + +MAINCLASS=edu.illinois.cs.cogcomp.srl.Main +#MAINCLASS=edu.illinois.cs.cogcomp.srl.SemanticRoleLabeler + +time nice java $OPTIONS $MAINCLASS "$@" diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/ColumnFormatWriter.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/ColumnFormatWriter.java new file mode 100644 index 000000000..35b2a6480 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/ColumnFormatWriter.java @@ -0,0 +1,432 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl; + +import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.*; +import edu.illinois.cs.cogcomp.core.datastructures.trees.Tree; +import edu.illinois.cs.cogcomp.edison.features.helpers.ParseHelper; +import edu.illinois.cs.cogcomp.edison.features.helpers.WordHelpers; +import edu.illinois.cs.cogcomp.nlp.utilities.ParseUtils; + +import java.io.PrintWriter; +import java.util.*; + +/** + * Prints text annotation formatted with one word per line as follows.
+ * form POS full-parse chunk NE verb-sense verb-lemma [verb1-args + * [verb2-args ... ]] + *
+ * + * @author Vivek Srikumar + */ +public class ColumnFormatWriter { + + private final String predicateArgumentViewName; + private final String parseViewName; + + public ColumnFormatWriter(String parseViewName, String predicateArgumentViewName) { + this.parseViewName = parseViewName; + this.predicateArgumentViewName = predicateArgumentViewName; + } + + public ColumnFormatWriter() { + this(ViewNames.PARSE_CHARNIAK, ViewNames.SRL_VERB); + } + + public void transform(Iterable reader, PrintWriter out) throws Exception { + + for (TextAnnotation ta : reader) { + transform(ta, out); + } + } + + public void transform(TextAnnotation ta, PrintWriter out) throws Exception { + String[][] columns = transformToColumns(ta); + printFormatted(columns, out, ta); + } + + public void printPredicateArgumentView(PredicateArgumentView pav, PrintWriter out) { + // System.out.println("*" + pav + "*"); + + List columns = new ArrayList<>(); + convertPredicateArgView(pav.getTextAnnotation(), pav, columns, false); + + String[][] tr = transpose(columns, pav.getTextAnnotation().size()); + + printFormatted(tr, out, pav.getTextAnnotation()); + } + + private void printFormatted(String[][] columns, PrintWriter out, TextAnnotation ta) { + // leftOfStar: length of everything before the asterisk. + // rightOfStar: length of asterisk and what comes after. + + int[] leftOfStar = new int[columns[0].length]; + int[] rightOfStart = new int[columns[0].length]; + + for (String[] rowData : columns) { + for (int col = 0; col < rowData.length; col++) { + + String word = rowData[col]; + + int starPos = word.indexOf("*"); + + int lenLeft, lenRight; + if (starPos < 0) { + lenLeft = word.length(); + lenRight = -1; + } else { + lenLeft = starPos + 1; + lenRight = word.length() - starPos + 1; + } + + if (leftOfStar[col] < lenLeft) + leftOfStar[col] = lenLeft; + + if (rightOfStart[col] < lenRight) + rightOfStart[col] = lenRight; + } + } + + // System.out.println("here"); + + assert ta.size() == columns.length; + + for (int sentenceId = 0; sentenceId < ta.getNumberOfSentences(); sentenceId++) { + + int start = ta.getSentence(sentenceId).getStartSpan(); + + for (int row = start; row < ta.getSentence(sentenceId).getEndSpan(); row++) { + String[] rowData = columns[row]; + + out.print(rowData[0]); + + // print the spaces + for (int spCount = rowData[0].length(); spCount < leftOfStar[0]; spCount++) + out.print(" "); + + out.print(" " + rowData[1]); + + // print the spaces + for (int spCount = rowData[1].length(); spCount < leftOfStar[1]; spCount++) + out.print(" "); + + out.print(" "); + + for (int colId = 2; colId < rowData.length; colId++) { + + String word = rowData[colId]; + + int starPos = word.indexOf("*"); + + int leftSpaces, rightSpaces; + leftSpaces = leftOfStar[colId]; + rightSpaces = rightOfStart[colId]; + + if (rightSpaces == 0) + leftSpaces = 0; + else + leftSpaces -= starPos; + + if (rightSpaces == 0) { + rightSpaces = leftOfStar[colId] - word.length(); + } else { + rightSpaces -= (word.length() - starPos); + } + + for (int i = 0; i < leftSpaces - 1; i++) + out.print(" "); + + out.print(word + " "); + + for (int i = 0; i < rightSpaces; i++) + out.print(" "); + + } + + out.println(); + } + + out.println(); + } + } + + private String[][] transpose(List columns, int size) { + String[][] output = new String[size][]; + + for (int i = 0; i < size; i++) { + output[i] = new String[columns.size()]; + } + + for (int row = 0; row < size; row++) { + for (int col = 0; col < columns.size(); col++) { + output[row][col] = columns.get(col)[row]; + } + } + + return output; + } + + /** + * Return a table. Numrows = number of words. Num Cols depends on how many predicate arg + * relations we have + */ + private String[][] transformToColumns(TextAnnotation ta) { + List columns = new ArrayList<>(); + + // first the words + String[] form = new String[ta.size()]; + String[] pos = new String[ta.size()]; + for (int i = 0; i < ta.size(); i++) { + form[i] = WordHelpers.getWord(ta, i); + pos[i] = WordHelpers.getPOS(ta, i); + } + + columns.add(form); + columns.add(pos); + + // now the parse + String[] parse = getParse(ta); + columns.add(parse); + + // add the chunks + String[] chunk = getChunkData(ta); + columns.add(chunk); + + // add the ner. For now, we don't have ner annotation + String[] ne = getNEData(ta); + columns.add(ne); + + // if (ta.hasView(ViewNames.SRL_NOM)) + // // add the predicate argument column information + // addPredicateArgs(columns, ta); + + // add the predicate argument column information + addPredicateArgs(columns, ta); + + return transpose(columns, ta.size()); + } + + private static String[] getNEData(TextAnnotation ta) { + + if (!ta.hasView(ViewNames.NER_CONLL)) { + String[] chunk = new String[ta.size()]; + + for (int i = 0; i < ta.size(); i++) { + chunk[i] = "*"; + } + + return chunk; + } + SpanLabelView nerView = (SpanLabelView) ta.getView(ViewNames.NER_CONLL); + + List nerConstituents = nerView.getConstituents(); + + Collections.sort(nerConstituents, TextAnnotationUtilities.constituentStartComparator); + + Map cc = new HashMap<>(); + for (Constituent c : nerConstituents) { + for (int i = c.getStartSpan(); i < c.getEndSpan(); i++) { + if (i == c.getStartSpan()) + cc.put(i, "(" + c.getLabel()); + else + cc.put(i, ""); + + cc.put(i, cc.get(i) + "*"); + + if (i == c.getEndSpan() - 1) + cc.put(i, cc.get(i) + ")"); + } + } + + String[] ner = new String[ta.size()]; + for (int i = 0; i < ta.size(); i++) { + if (cc.containsKey(i)) { + ner[i] = cc.get(i); + } else + ner[i] = "*"; + } + return ner; + } + + private static String[] getChunkData(TextAnnotation ta) { + + if (!ta.hasView(ViewNames.SHALLOW_PARSE)) { + String[] chunk = new String[ta.size()]; + + for (int i = 0; i < ta.size(); i++) { + chunk[i] = "*"; + } + + return chunk; + } + + SpanLabelView chunkView = (SpanLabelView) ta.getView(ViewNames.SHALLOW_PARSE); + + List chunkConstituents = chunkView.getConstituents(); + + Collections.sort(chunkConstituents, TextAnnotationUtilities.constituentStartComparator); + + Map cc = new HashMap<>(); + for (Constituent c : chunkConstituents) { + for (int i = c.getStartSpan(); i < c.getEndSpan(); i++) { + if (i == c.getStartSpan()) + cc.put(i, "(" + c.getLabel()); + else + cc.put(i, ""); + + cc.put(i, cc.get(i) + "*"); + + if (i == c.getEndSpan() - 1) + cc.put(i, cc.get(i) + ")"); + } + } + + String[] chunk = new String[ta.size()]; + for (int i = 0; i < ta.size(); i++) { + if (cc.containsKey(i)) { + chunk[i] = cc.get(i); + } else + chunk[i] = "*"; + } + return chunk; + } + + private void addPredicateArgs(List columns, TextAnnotation ta) { + PredicateArgumentView predArgView = null; + + if (ta.hasView(predicateArgumentViewName)) + predArgView = (PredicateArgumentView) ta.getView(predicateArgumentViewName); + + convertPredicateArgView(ta, predArgView, columns, true); + + } + + private void convertPredicateArgView(TextAnnotation ta, PredicateArgumentView pav, + List columns, boolean addSense) { + + List predicates = new ArrayList<>(); + if (pav != null) + predicates = pav.getPredicates(); + + Collections.sort(predicates, TextAnnotationUtilities.constituentStartComparator); + + int size = ta.size(); + + addPredicateInfo(columns, predicates, size, addSense); + + for (Constituent predicate : predicates) { + assert pav != null; + List args = pav.getArguments(predicate); + + String[] paInfo = addPredicateArgInfo(predicate, args, size); + + columns.add(paInfo); + } + } + + private void addPredicateInfo(List columns, List predicates, int size, + boolean addSense) { + Map senseMap = new HashMap<>(); + Map lemmaMap = new HashMap<>(); + + for (Constituent c : predicates) { + senseMap.put(c.getStartSpan(), c.getAttribute(PredicateArgumentView.SenseIdentifer)); + lemmaMap.put(c.getStartSpan(), c.getAttribute(PredicateArgumentView.LemmaIdentifier)); + } + + String[] sense = new String[size]; + String[] lemma = new String[size]; + + for (int i = 0; i < size; i++) { + if (lemmaMap.containsKey(i)) { + + sense[i] = senseMap.get(i); + lemma[i] = lemmaMap.get(i); + } else { + sense[i] = "-"; + lemma[i] = "-"; + } + } + + if (addSense) + columns.add(sense); + columns.add(lemma); + } + + private String[] addPredicateArgInfo(Constituent predicate, List args, int size) { + Map paInfo = new HashMap<>(); + + paInfo.put(predicate.getStartSpan(), "(V*)"); + for (Relation r : args) { + String argPredicate = r.getRelationName(); + + argPredicate = argPredicate.replaceAll("ARG", "A"); + argPredicate = argPredicate.replaceAll("Support", "SUP"); + + for (int i = r.getTarget().getStartSpan(); i < r.getTarget().getEndSpan(); i++) { + paInfo.put(i, "*"); + if (i == r.getTarget().getStartSpan()) + paInfo.put(i, "(" + argPredicate + paInfo.get(i)); + if (i == r.getTarget().getEndSpan() - 1) + paInfo.put(i, paInfo.get(i) + ")"); + + } + } + + String[] paColumn = new String[size]; + for (int i = 0; i < size; i++) { + if (paInfo.containsKey(i)) + paColumn[i] = paInfo.get(i); + else + paColumn[i] = "*"; + + } + + return paColumn; + } + + private String[] getParse(TextAnnotation ta) { + String[] parse = new String[ta.size()]; + + for (int sentenceId = 0; sentenceId < ta.getNumberOfSentences(); sentenceId++) { + + Tree tree = ParseHelper.getParseTree(parseViewName, ta, sentenceId); + + Sentence sentence = ta.getSentence(sentenceId); + + tree = ParseUtils.snipNullNodes(tree); + tree = ParseUtils.stripFunctionTags(tree); + + String[] treeLines = tree.toString().split("\n"); + + if (treeLines.length != sentence.size()) { + + System.out.println(ta); + System.out.println(ta.getView(parseViewName)); + + System.out.println("Sentence: " + sentence); + + throw new IllegalStateException("Expected " + sentence.size() + + " tokens, but found " + treeLines.length + " tokens in the tree"); + } + + for (int i = 0; i < treeLines.length; i++) { + + String t = treeLines[i].replaceAll(" ", ""); + + // get rid of the word + t = t.replaceAll("\\([^\\(\\)]*\\)", "*"); + + // get rid of the pos and replace with a "*" + parse[sentence.getStartSpan() + i] = t; + } + } + return parse; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/Constants.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/Constants.java new file mode 100644 index 000000000..c213acd36 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/Constants.java @@ -0,0 +1,20 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl; + +/** + * @author Vivek Srikumar TODO Change this before shipping + */ +public class Constants { + public final static String systemVersion = "5.4.1"; + + public final static String systemName = "illinoisSRL"; + + public final static String verbSRLSystemIdentifier = systemName + "-verb" + systemVersion; + public final static String nomSRLSystemIdentifier = systemName + "-nom" + systemVersion; +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/Main.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/Main.java new file mode 100644 index 000000000..d1ce149eb --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/Main.java @@ -0,0 +1,594 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl; + +import edu.illinois.cs.cogcomp.core.datastructures.Lexicon; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.IResetableIterator; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.experiments.ClassificationTester; +import edu.illinois.cs.cogcomp.core.io.IOUtils; +import edu.illinois.cs.cogcomp.core.io.caches.TextAnnotationCache; +import edu.illinois.cs.cogcomp.core.io.caches.TextAnnotationDBHandler; +import edu.illinois.cs.cogcomp.core.io.caches.TextAnnotationMapDBHandler; +import edu.illinois.cs.cogcomp.core.stats.Counter; +import edu.illinois.cs.cogcomp.core.utilities.commands.CommandDescription; +import edu.illinois.cs.cogcomp.core.utilities.commands.CommandIgnore; +import edu.illinois.cs.cogcomp.core.utilities.commands.InteractiveShell; +import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager; +import edu.illinois.cs.cogcomp.infer.ilp.ILPSolverFactory; +import edu.illinois.cs.cogcomp.nlp.corpusreaders.NombankReader; +import edu.illinois.cs.cogcomp.nlp.corpusreaders.PropbankReader; +import edu.illinois.cs.cogcomp.sl.core.SLParameters; +import edu.illinois.cs.cogcomp.sl.core.SLProblem; +import edu.illinois.cs.cogcomp.sl.learner.Learner; +import edu.illinois.cs.cogcomp.sl.learner.LearnerFactory; +import edu.illinois.cs.cogcomp.sl.util.IFeatureVector; +import edu.illinois.cs.cogcomp.sl.util.WeightVector; +import edu.illinois.cs.cogcomp.srl.caches.FeatureVectorCacheFile; +import edu.illinois.cs.cogcomp.srl.core.ModelInfo; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.core.SRLType; +import edu.illinois.cs.cogcomp.srl.data.Dataset; +import edu.illinois.cs.cogcomp.srl.experiment.PreExtractor; +import edu.illinois.cs.cogcomp.srl.experiment.PruningPreExtractor; +import edu.illinois.cs.cogcomp.srl.experiment.TextPreProcessor; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPInference; +import edu.illinois.cs.cogcomp.srl.inference.SRLMulticlassInference; +import edu.illinois.cs.cogcomp.srl.learn.SRLFeatureExtractor; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassLabel; +import edu.illinois.cs.cogcomp.srl.learn.IdentifierThresholdTuner; +import edu.illinois.cs.cogcomp.srl.nom.NomSRLManager; +import edu.illinois.cs.cogcomp.srl.utilities.WeightVectorUtils; +import edu.illinois.cs.cogcomp.srl.verb.VerbSRLManager; +import edu.illinois.cs.cogcomp.core.experiments.evaluators.PredicateArgumentEvaluator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.*; + +public class Main { + private final static Logger log = LoggerFactory.getLogger(Main.class); + + private static String defaultParser; + private static SRLProperties properties; + private static String configFile; + + + @CommandIgnore + public static void main(String[] arguments) { + + InteractiveShell
shell = new InteractiveShell<>(Main.class); + + if (arguments.length == 0) { + System.err.println("Usage: command"); + System.err.println("Required parameter config-file missing."); + shell.showDocumentation(); + } else if (arguments.length == 1) { + System.err.println("Usage: command"); + shell.showDocumentation(); + } else { + long start_time = System.currentTimeMillis(); + try { + configFile = arguments[0]; + SRLProperties.initialize(configFile); + properties = SRLProperties.getInstance(); + + defaultParser = SRLProperties.getInstance().getDefaultParser(); + + String[] args = new String[arguments.length - 1]; + System.arraycopy(arguments, 1, args, 0, args.length); + shell.runCommand(args); + + long runTime = (System.currentTimeMillis() - start_time) / 1000; + System.out.println("This experiment took " + runTime + " secs"); + + } catch (AssertionError e) { + e.printStackTrace(); + System.exit(-1); + + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + @CommandDescription( + description = "Performs the full training & testing sequence for all SRL types", + usage = "expt [Verb | Nom] cacheDatasets=[true | false]") + public static void expt(String srlType, String cacheDatasets) throws Exception { + // Step 1: Cache all the datasets we're going to use + if (Boolean.parseBoolean(cacheDatasets)) + cacheDatasets(); + + // Step 2: Iterate between pre-extracting all the features needed for training and training + + // We don't need to train a predicate classifier for Verb + if (SRLType.valueOf(srlType) == SRLType.Nom) { + preExtract(srlType, "Predicate"); + train(srlType, "Predicate"); + } + + preExtract(srlType, "Sense"); + train(srlType, "Sense"); + + preExtract(srlType, "Identifier"); + train(srlType, "Identifier"); + tuneIdentifier(srlType); + + preExtract(srlType, "Classifier"); + train(srlType, "Classifier"); + + // Step 3: Evaluate + evaluate(srlType); + } + + @CommandDescription(description = "Reads and caches all the datasets", usage = "cacheDatasets") + public static void cacheDatasets() throws Exception { + log.info("Initializing datasets"); + TextAnnotationCache taCache = getTextAnnotationCache(); + + // Add Propbank data + log.info("Caching PropBank data"); + cacheVerbNom(SRLType.Verb, taCache); + + log.info("Caching NomBank data"); + cacheVerbNom(SRLType.Nom, taCache); + + log.info("Cached all datasets"); + + log.info("Adding required views in PTB"); + addRequiredViews(taCache.getDataset(Dataset.PTBAll.name()), taCache); + } + + private static void cacheVerbNom(SRLType srlType, TextAnnotationCache taCache) throws Exception { + String treebankHome = properties.getPennTreebankHome(); + String[] allSectionsArray = properties.getAllSections(); + List trainSections = Arrays.asList(properties.getAllTrainSections()); + List testSections = Collections.singletonList(properties.getTestSections()); + List trainDevSections = Arrays.asList(properties.getTrainDevSections()); + List devSections = Arrays.asList(properties.getDevSections()); + List ptb0204Sections = Arrays.asList("02", "03", "04"); + + String dataHome; + if (srlType == SRLType.Verb) + dataHome = properties.getPropbankHome(); + else + dataHome = properties.getNombankHome(); + + String goldView = (srlType == SRLType.Verb) ? ViewNames.SRL_VERB : ViewNames.SRL_NOM; + goldView += "_GOLD"; + Iterator data; + if (srlType == SRLType.Verb) + data = new PropbankReader(treebankHome, dataHome, allSectionsArray, goldView, true); + else + data = new NombankReader(treebankHome, dataHome, allSectionsArray, goldView, true); + + int count = 0; + while (data.hasNext()) { + TextAnnotation ta = data.next(); + if (ta.hasView(goldView)) { + String id = ta.getId(); + String section = id.substring(id.indexOf('/') + 1, id.lastIndexOf('/')); + taCache.addTextAnnotation(Dataset.PTBAll.name(), ta); + if (trainSections.contains(section)) + taCache.addTextAnnotation(Dataset.PTBTrain.name(), ta); + if (devSections.contains(section)) + taCache.addTextAnnotation(Dataset.PTBDev.name(), ta); + if (trainDevSections.contains(section)) + taCache.addTextAnnotation(Dataset.PTBTrainDev.name(), ta); + if (testSections.contains(section)) + taCache.addTextAnnotation(Dataset.PTBTest.name(), ta); + if (ptb0204Sections.contains(section)) + taCache.addTextAnnotation(Dataset.PTB0204.name(), ta); + } + + count++; + if (count % 10000 == 0) + System.out.println(count + " sentences done"); + } + taCache.close(); + } + + private static void addRequiredViews(IResetableIterator dataset, TextAnnotationCache taCache) + throws IOException { + Counter addedViews = new Counter<>(); + log.info("Initializing pre-processor"); + TextPreProcessor.initialize(new ResourceManager(configFile)); + + int count = 0; + while (dataset.hasNext()) { + TextAnnotation ta = dataset.next(); + Set views = new HashSet<>(ta.getAvailableViews()); + + try { + TextPreProcessor.getInstance().preProcessText(ta); + } catch (Exception e) { + // Remove from dataset + log.error("Annotation failed, removing sentence from dataset"); + taCache.removeTextAnnotation(ta); + continue; + } + String parserView = ViewNames.DEPENDENCY + ":"; + String parser = properties.getDefaultParser(); + if (parser.equals("Charniak")) + parserView += ViewNames.PARSE_CHARNIAK; + if (parser.equals("Berkeley")) + parserView += ViewNames.PARSE_BERKELEY; + if (parser.equals("Stanford")) + parserView += ViewNames.PARSE_STANFORD; + if (ta.getView(parserView).getNumberOfConstituents() != ta.getSentence(0).size()) { + log.error("Head-dependency mismatch, removing sentence from dataset"); + taCache.removeTextAnnotation(ta); + continue; + } + + Set newViews = new HashSet<>(ta.getAvailableViews()); + newViews.removeAll(views); + + if (newViews.size() > 0) { + taCache.updateTextAnnotation(ta); + for (String s : newViews) + addedViews.incrementCount(s); + } + count++; + if (count % 1000 == 0) + log.info(count + " sentences done"); + } + log.info("New views: "); + for (String s : addedViews.items()) + log.info(s + "\t" + addedViews.getCount(s)); + taCache.close(); + } + + @CommandIgnore + static SRLManager getManager(SRLType srlType, boolean trainingMode) throws Exception { + String viewName; + if (defaultParser == null) + defaultParser = SRLProperties.getInstance().getDefaultParser(); + switch (defaultParser) { + case "Charniak": + viewName = ViewNames.PARSE_CHARNIAK; + break; + case "Berkeley": + viewName = ViewNames.PARSE_BERKELEY; + break; + case "Stanford": + viewName = ViewNames.PARSE_STANFORD; + break; + default: + viewName = defaultParser; + break; + } + + if (srlType == SRLType.Verb) + return new VerbSRLManager(trainingMode, viewName); + else if (srlType == SRLType.Nom) + return new NomSRLManager(trainingMode, viewName); + else + return null; + } + + @CommandDescription( + description = "Pre-extracts the features for a specific model and SRL type. " + + "Run this before training", + usage = "preExtract [Verb | Nom] [Predicate | Sense | Identifier | Classifier]") + public static void preExtract(String srlType_, String model) throws Exception { + SRLType srlType = SRLType.valueOf(srlType_); + SRLManager manager = getManager(srlType, true); + + String gapStr = ""; + for (int i = 0; i < ((29 - model.length()) / 2); i++) + gapStr += " "; + log.info("\n\n\n\n" + "**************************************************\n" + "** " + + gapStr + "PRE-EXTRACTING " + model.toUpperCase() + gapStr + " **\n" + + "**************************************************\n"); + // If models directory doesn't exist create it + if (!IOUtils.isDirectory(SRLProperties.getInstance().getModelsDir())) + IOUtils.mkdir(SRLProperties.getInstance().getModelsDir()); + + int numConsumers = Runtime.getRuntime().availableProcessors(); + + Models modelToExtract = Models.valueOf(model); + + Dataset dataset = Dataset.PTBTrainDev; + if (modelToExtract == Models.Identifier) + dataset = Dataset.PTBTrain; + + + log.info("Pre-extracting {} features", modelToExtract); + assert manager != null; + ModelInfo modelInfo = manager.getModelInfo(modelToExtract); + + String featureSet = "" + modelInfo.featureManifest.getIncludedFeatures().hashCode(); + + String allDataCacheFile = + properties.getFeatureCacheFile(srlType, modelToExtract, featureSet, defaultParser, + dataset); + FeatureVectorCacheFile featureCache = + preExtract(numConsumers, manager, modelToExtract, dataset, allDataCacheFile, false); + + pruneFeatures(numConsumers, manager, modelToExtract, featureCache, + properties.getPrunedFeatureCacheFile(srlType, modelToExtract, featureSet, + defaultParser)); + + Lexicon lexicon = + modelInfo.getLexicon().getPrunedLexicon(manager.getPruneSize(modelToExtract)); + + log.info("Saving lexicon with {} features to {}", lexicon.size(), + manager.getLexiconFileName(modelToExtract)); + log.info(lexicon.size() + " features in the lexicon"); + + lexicon.save(manager.getLexiconFileName(modelToExtract)); + + // We don't have a separate dev set to tune Identifier for prepSRL + if (modelToExtract == Models.Identifier) { + Dataset datasetIdentifier = Dataset.PTBDev; + String devCacheFile = + properties.getFeatureCacheFile(srlType, modelToExtract, featureSet, + defaultParser, datasetIdentifier); + preExtract(numConsumers, manager, modelToExtract, datasetIdentifier, devCacheFile, + false); + } + } + + private static void pruneFeatures(int numConsumers, SRLManager manager, Models modelToExtract, + FeatureVectorCacheFile featureCache, String cacheFile2) throws Exception { + if (IOUtils.exists(cacheFile2)) { + log.warn("Old pruned cache file found. Not doing anything..."); + return; + } + + log.info("Pruning features. Saving pruned features to {}", cacheFile2); + + FeatureVectorCacheFile prunedfeatureCache = + new FeatureVectorCacheFile(cacheFile2, modelToExtract, manager); + + PruningPreExtractor p1 = + new PruningPreExtractor(manager, modelToExtract, featureCache, prunedfeatureCache, + numConsumers); + p1.run(); + p1.finalize(); + } + + private static FeatureVectorCacheFile preExtract(int numConsumers, SRLManager manager, + Models modelToExtract, Dataset dataset, String cacheFile, boolean lockLexicon) + throws Exception { + if (IOUtils.exists(cacheFile)) { + log.warn("Old cache file found. Returning it..."); + FeatureVectorCacheFile vectorCacheFile = + new FeatureVectorCacheFile(cacheFile, modelToExtract, manager); + vectorCacheFile.openReader(); + while (vectorCacheFile.hasNext()) { + Pair pair = vectorCacheFile.next(); + IFeatureVector cachedFeatureVector = + pair.getFirst().getCachedFeatureVector(modelToExtract); + int length = cachedFeatureVector.getNumActiveFeatures(); + for (int i = 0; i < length; i++) { + manager.getModelInfo(modelToExtract).getLexicon() + .countFeature(cachedFeatureVector.getIdx(i)); + } + } + vectorCacheFile.close(); + vectorCacheFile.openReader(); + return vectorCacheFile; + } + TextAnnotationCache taCache = getTextAnnotationCache(); + FeatureVectorCacheFile featureCache = + new FeatureVectorCacheFile(cacheFile, modelToExtract, manager); + Iterator data = taCache.getDataset(dataset.name()); + PreExtractor p = new PreExtractor(manager, data, numConsumers, modelToExtract, featureCache); + taCache.close(); + + if (lockLexicon) + p.lockLexicon(); + + p.run(); + + p.finalize(); + return featureCache; + } + + @CommandDescription(description = "Trains a specific model and SRL type", + usage = "train [Verb | Nom] [Predicate | Sense | Identifier | Classifier]") + public static void train(String srlType_, String model_) throws Exception { + SRLType srlType = SRLType.valueOf(srlType_); + SRLManager manager = getManager(srlType, true); + + String gapStr = ""; + for (int i = 0; i < ((36 - model_.length()) / 2); i++) + gapStr += " "; + log.info("\n\n\n\n" + "**************************************************\n" + "** " + + gapStr + "TRAINING " + model_.toUpperCase() + gapStr + " **\n" + + "**************************************************\n"); + + Models model = Models.valueOf(model_); + assert manager != null; + ModelInfo modelInfo = manager.getModelInfo(model); + + String featureSet = "" + modelInfo.featureManifest.getIncludedFeatures().hashCode(); + String cacheFile = + properties.getPrunedFeatureCacheFile(srlType, model, featureSet, defaultParser); + System.out.println("In train feat cahce is " + cacheFile); + + // NB: Tuning code for the C value has been deleted + double c = 0.01; + FeatureVectorCacheFile cache; + + if (model == Models.Classifier) + c = 0.00390625; + + cache = new FeatureVectorCacheFile(cacheFile, model, manager); + SLProblem problem; + problem = cache.getStructuredProblem(); + + cache.close(); + + log.info("Setting up solver, learning may take time if you have too many instances in SLProblem ...."); + + SLParameters params = new SLParameters(); + params.loadConfigFile(properties.getLearnerConfig()); + params.C_FOR_STRUCTURE = (float) c; + + SRLMulticlassInference infSolver = new SRLMulticlassInference(manager, model); + Learner learner = LearnerFactory.getLearner(infSolver, new SRLFeatureExtractor(), params); + WeightVector w = learner.train(problem); + WeightVectorUtils.save(manager.getModelFileName(model), w); + } + + private static void tuneIdentifier(String srlType_) throws Exception { + SRLType srlType = SRLType.valueOf(srlType_); + SRLManager manager = getManager(srlType, true); + + int nF = 2; + if (srlType == SRLType.Nom) + nF = 3; + + assert manager != null; + ModelInfo modelInfo = manager.getModelInfo(Models.Identifier); + modelInfo.loadWeightVector(); + + String featureSet = "" + modelInfo.featureManifest.getIncludedFeatures().hashCode(); + + Dataset dataset = Dataset.PTBDev; + String cacheFile = + properties.getFeatureCacheFile(srlType, Models.Identifier, featureSet, + defaultParser, dataset); + + FeatureVectorCacheFile cache = + new FeatureVectorCacheFile(cacheFile, Models.Identifier, manager); + + SLProblem problem = cache.getStructuredProblem(); + cache.close(); + + IdentifierThresholdTuner tuner = new IdentifierThresholdTuner(manager, nF, problem); + + List A = new ArrayList<>(); + List B = new ArrayList<>(); + + for (double x = 0.01; x < 10; x += 0.01) { + A.add(x); + B.add(x); + } + + Pair pair = tuner.tuneIdentifierScale(A, B); + manager.writeIdentifierScale(pair.getFirst(), pair.getSecond()); + } + + @CommandDescription(description = "Performs evaluation.", usage = "evaluate [Verb | Nom]") + public static void evaluate(String srlType_) throws Exception { + SRLType srlType = SRLType.valueOf(srlType_); + SRLManager manager = getManager(srlType, false); + + Dataset testSet = Dataset.PTBTest; + + ILPSolverFactory solver = new ILPSolverFactory(properties.getILPSolverType(true)); + + String outDir = properties.getOutputDir(); + PrintWriter goldWriter = null, predWriter = null; + ColumnFormatWriter writer = null; + assert manager != null; + String goldOutFile = null, predOutFile = null; + if (outDir != null) { + // If output directory doesn't exist, create it + if (!IOUtils.isDirectory(outDir)) + IOUtils.mkdir(outDir); + + String outputFilePrefix = + outDir + "/" + srlType + "." + manager.defaultParser + "." + + new Random().nextInt(); + + goldOutFile = outputFilePrefix + ".gold"; + goldWriter = new PrintWriter(new File(goldOutFile)); + predOutFile = outputFilePrefix + ".predicted"; + predWriter = new PrintWriter(new File(predOutFile)); + writer = new ColumnFormatWriter(); + } + + ClassificationTester tester = new ClassificationTester(); + tester.ignoreLabelFromSummary("V"); + + + ClassificationTester senseTester = new ClassificationTester(); + + long start = System.currentTimeMillis(); + int count = 0; + + manager.getModelInfo(Models.Identifier).loadWeightVector(); + manager.getModelInfo(Models.Classifier).loadWeightVector(); + manager.getModelInfo(Models.Sense).loadWeightVector(); + log.info("All models weights loaded now!"); + + TextAnnotationCache taCache = getTextAnnotationCache(); + IResetableIterator dataset = taCache.getDataset(testSet.name()); + PredicateArgumentEvaluator evaluator = new PredicateArgumentEvaluator(); + + while (dataset.hasNext()) { + TextAnnotation ta = dataset.next(); + + if (!ta.hasView(manager.getGoldViewName())) + continue; + + PredicateArgumentView gold = + (PredicateArgumentView) ta.getView(manager.getGoldViewName()); + + SRLILPInference inference = manager.getInference(solver, gold.getPredicates()); + + assert inference != null; + PredicateArgumentView prediction = inference.getOutputView(); + + evaluator.evaluate(tester, gold, prediction); + evaluator.evaluateSense(senseTester, gold, prediction); + + if (outDir != null) { + writer.printPredicateArgumentView(gold, goldWriter); + } + + count++; + if (count % 1000 == 0) { + long end = System.currentTimeMillis(); + log.info(count + " sentences done. Took " + (end - start) + "ms, " + "F1 so far = " + + tester.getMacroF1()); + } + } + long end = System.currentTimeMillis(); + System.out.println(count + " sentences done. Took " + (end - start) + "ms"); + + System.out.println("* Arguments"); + System.out.println(tester.getPerformanceTable(false).toOrgTable()); + + System.out.println("\n\n* Sense"); + System.out.println(senseTester.getPerformanceTable(false).toOrgTable()); + + if (outDir != null) { + goldWriter.close(); + predWriter.close(); + + System.out.println("To use standard CoNLL evaluation, compare the following files:\n" + + goldOutFile + " " + predOutFile); + } + taCache.close(); + } + + private static TextAnnotationCache getTextAnnotationCache() { + TextAnnotationCache taCache; + if (properties.getDatasetCache().equals("H2")) + taCache = new TextAnnotationDBHandler(properties.getSentenceDBFile(), Dataset.stringValues()); + else if (properties.getDatasetCache().equals("MapDB")) + taCache = new TextAnnotationMapDBHandler(properties.getSentenceDBFile()); + else throw new RuntimeException("Unknown Dataset Cache type"); + return taCache; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/SRLProperties.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/SRLProperties.java new file mode 100644 index 000000000..04274aee4 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/SRLProperties.java @@ -0,0 +1,207 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl; + +import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager; +import edu.illinois.cs.cogcomp.infer.ilp.ILPSolverFactory; +import edu.illinois.cs.cogcomp.srl.config.SrlConfigurator; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLType; +import edu.illinois.cs.cogcomp.srl.data.Dataset; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.naming.NamingEnumeration; +import java.io.IOException; +import java.util.Properties; + +public class SRLProperties { + private static final Logger log = LoggerFactory.getLogger(SRLProperties.class); + private static final String NAME = SRLProperties.class.getCanonicalName(); + private static SRLProperties theInstance; + private ResourceManager config; + + /** + * configFile must have all parameters set, ideally using the SrlConfigurator class. + * + * @param configFile file with configuration parameters + */ + + private SRLProperties(String configFile) throws IOException { + this(new ResourceManager(configFile)); + } + + /** + * ResourceManager must have all parameters set, ideally using the SrlConfigurator class. + * + * @param rm + */ + private SRLProperties(ResourceManager rm) { + config = rm; + } + + /** + * If SRLProperties has not yet been instantiated, initialize an instance with default values. + * + * @throws Exception + */ + public static void initialize() throws Exception { + initialize(new ResourceManager(new Properties())); + } + + /** + * If SRLProperties has not yet been instantiated, initialize new instance with the non-default + * parameters specified in the configFile named in the argument, and default parameters + * otherwise. If a null string is given as the argument, all default parameters are used. + * + * @param configFile if non-null, names a file in which non-default parameters are specified. + * @throws Exception + */ + public static void initialize(String configFile) throws Exception { + initialize(new ResourceManager(configFile)); + } + + public static void initialize(ResourceManager rm) { + log.info("## initializing {}.", NAME); + ResourceManager fullRm = new SrlConfigurator().getConfig(rm); + theInstance = new SRLProperties(fullRm); + } + + + public static SRLProperties getInstance(ResourceManager rm) { + if (theInstance == null) + try { + initialize(rm); + } catch (Exception e) { + e.printStackTrace(); + } + return theInstance; + } + + public static SRLProperties getInstance() { + if (theInstance == null) { + System.out + .println("SRL config not initialized. Instantiating with default parameters."); + try { + initialize(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return theInstance; + } + + public ResourceManager getConfig() { + return config; + } + + public boolean useCurator() { + return config.getBoolean("UseCurator"); + } + + String getFeatureCacheFile(SRLType SRLType, Models type, String featureSet, String parser, + Dataset dataset) { + return this.config.getString("CacheDirectory") + "/features." + SRLType.name() + "." + + type.name() + "." + featureSet + "." + dataset + "." + parser + ".cache"; + } + + String getPrunedFeatureCacheFile(SRLType SRLType, Models type, String featureSet, String parser) { + return this.config.getString("CacheDirectory") + "/features." + SRLType.name() + "." + + type.name() + "." + featureSet + "." + parser + ".pruned.cache"; + } + + public String getPennTreebankHome() { + return config.getString("PennTreebankHome"); + } + + public int getNumFeatExtThreads() { + return config.getInt("NumFeatExtThreads"); + } + + public String[] getAllTrainSections() { + return new String[] {"02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", + "13", "14", "15", "16", "17", "18", "19", "20", "21", "22"}; + } + + public String getTestSections() { + return "23"; + } + + public String[] getAllSections() { + return new String[] {"02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", + "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "24", "23"}; + + } + + public String[] getTrainDevSections() { + return new String[] {"02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", + "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "24"}; + } + + public String[] getDevSections() { + return new String[] {"24"}; + } + + public String getPropbankHome() { + return config.getString("PropbankHome"); + } + + public String getNombankHome() { + return config.getString("NombankHome"); + } + + public String getSentenceDBFile() { + return this.config.getString("CacheDirectory") + "/sentences.db"; + } + + public String getDefaultParser() { + return config.getString("DefaultParser"); + } + + public String getModelsDir() { + return config.getString("ModelsDirectory"); + } + + public String getOutputDir() { + if (config.containsKey("OutputDirectory")) + return config.getString("OutputDirectory"); + return null; + } + + public String getSRLVersion() { + return Constants.systemVersion; + } + + public String getLearnerConfig() { + return this.config.getString("LearnerConfig"); + } + + public String getPipelineConfig() { + return this.config.getString("PipelineConfig"); + } + + public ILPSolverFactory.SolverType getILPSolverType(boolean isEvaluating) { + String solver = config.getString("ILPSolver"); + switch (solver) { + case "Gurobi": + if (isEvaluating) + return ILPSolverFactory.SolverType.JLISCuttingPlaneGurobi; + else + return ILPSolverFactory.SolverType.Gurobi; + case "OJAlgo": + return ILPSolverFactory.SolverType.OJAlgo; + default: + log.info("Using default ILP Solver: OJAlgo"); + return ILPSolverFactory.SolverType.OJAlgo; + } + } + + public String getDatasetCache() { + return config.getString("DatasetCache"); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/SemanticRoleLabeler.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/SemanticRoleLabeler.java new file mode 100644 index 000000000..5df3da37b --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/SemanticRoleLabeler.java @@ -0,0 +1,264 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl; + +import edu.illinois.cs.cogcomp.annotation.Annotator; +import edu.illinois.cs.cogcomp.annotation.AnnotatorException; +import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.View; +import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager; +import edu.illinois.cs.cogcomp.edison.annotators.ClauseViewGenerator; +import edu.illinois.cs.cogcomp.infer.ilp.ILPSolverFactory; +import edu.illinois.cs.cogcomp.srl.config.SrlConfigurator; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.core.SRLType; +import edu.illinois.cs.cogcomp.srl.experiment.TextPreProcessor; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPInference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +public class SemanticRoleLabeler extends Annotator { + private final static Logger log = LoggerFactory.getLogger(SemanticRoleLabeler.class); + public SRLManager manager; + private static SRLProperties properties; + + + public static void main(String[] arguments) { + if (arguments.length < 1) { + System.err.println("Usage: [Verb | Nom]"); + System.exit(-1); + } + String configFile = arguments[0]; + ResourceManager rm = null; + try { + rm = new SrlConfigurator().getConfig(new ResourceManager(configFile)); + } catch (IOException e) { + e.printStackTrace(); + System.exit(-1); + } + String srlType; + // If no second argument is provided it means we need all the SRL types + srlType = arguments.length == 1 ? null : arguments[1]; + + String input; + List srlLabelers = new ArrayList<>(); + + Properties props = new Properties(); + props.setProperty(SrlConfigurator.SRL_TYPE.key, SRLType.Verb.name()); + props.setProperty(SrlConfigurator.INSTANTIATE_PREPROCESSOR.key, SrlConfigurator.TRUE); + SrlConfigurator.mergeProperties(rm, new ResourceManager(props)); + + try { + if (srlType != null) + srlLabelers.add(new SemanticRoleLabeler(rm)); + else { + for (SRLType type : SRLType.values()) { + // srlType = type.name(); + props.setProperty(SrlConfigurator.SRL_TYPE.key, type.name()); + SrlConfigurator.mergeProperties(rm, new ResourceManager(props)); + + srlLabelers.add(new SemanticRoleLabeler(rm)); + } + } + } catch (Exception e) { + log.error("Unable to initialize SemanticRoleLabeler:"); + e.printStackTrace(); + System.exit(-1); + } + + do { + System.out.print("Enter text (underscore to quit): "); + input = System.console().readLine().trim(); + if (input.equals("_")) + return; + + if (!input.isEmpty()) { + // XXX Assuming that all SRL types require the same views + TextAnnotation ta; + try { + ta = TextPreProcessor.getInstance().preProcessText(input); + } catch (Exception e) { + log.error("Unable to pre-process the text:"); + e.printStackTrace(); + continue; + } + + for (SemanticRoleLabeler srl : srlLabelers) { + if (srlLabelers.size() > 1) + System.out.println(srl.getViewName()); + + PredicateArgumentView p; + try { + p = srl.getSRL(ta); + } catch (Exception e) { + log.error("Unable to produce SRL annotation:"); + e.printStackTrace(); + continue; + } + + System.out.println(p); + System.out.println(); + } + } + } while (!input.equals("_")); + } + + /** + * default loads Verb srl + * + * @throws Exception + */ + public SemanticRoleLabeler() throws Exception { + this(new ResourceManager(new Properties())); + } + + public SemanticRoleLabeler(ResourceManager rm) throws Exception { + this(rm, false); + } + + /** + * @param rm fully populated configuration (all fields from SrlConfigurator must be set) + * @param lazilyInitialize if 'true', defer loading resources until getView() is called + * @throws Exception + */ + public SemanticRoleLabeler(ResourceManager rm, boolean lazilyInitialize) throws Exception { + this(new SrlConfigurator().getConfig(rm), lazilyInitialize, false); + } + + /** + * protected because ResourceManager argument must have all entries in SrlConfigurator set + * before super() is called. + * + * @param config fully populated configuration (all fields from SrlConfigurator must be set) + * @param lazilyInitialize if 'true', defer loading resources until getView() is called + * @param irrelevantFlag a spurious argument to distinguish this protected constructor + */ + protected SemanticRoleLabeler(ResourceManager config, boolean lazilyInitialize, + boolean irrelevantFlag) { + super(getViewNameForType(config.getString(SrlConfigurator.SRL_TYPE.key)), + TextPreProcessor.requiredViews, lazilyInitialize, config); + } + + // public SemanticRoleLabeler(ResourceManager rm, String srlType, boolean isLazilyInitialized ) + // throws Exception { + // super(getViewNameForType(srlType), TextPreProcessor.requiredViews, isLazilyInitialized, new + // SrlConfigurator().getConfig(rm)); + // } + + @Override + public void initialize(ResourceManager rm) { + SRLProperties.initialize(rm); + properties = SRLProperties.getInstance(); + boolean initialize = rm.getBoolean(SrlConfigurator.INSTANTIATE_PREPROCESSOR.key); + + if (initialize) { + TextPreProcessor.initialize(properties); + } + + String srlType = rm.getString(SrlConfigurator.SRL_TYPE); + try { + manager = Main.getManager(SRLType.valueOf(srlType), false); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // log.info("Loading models"); + try { + loadModels(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static String getViewNameForType(String srlType) { + if (srlType.equals(SRLType.Verb.name())) + return ViewNames.SRL_VERB; + else if (srlType.equals(SRLType.Nom.name())) + return ViewNames.SRL_NOM; + else + throw new IllegalArgumentException("ERROR: type '" + srlType + "' not recognized."); + } + + public String getSRLCuratorName() { + return manager.getSRLSystemIdentifier(); + } + + public String getVersion() { + return properties.getSRLVersion(); + } + + private void loadModels() throws Exception { + for (Models m : Models.values()) { + if (manager.getSRLType() == SRLType.Verb && m == Models.Predicate) + continue; + + log.info("Loading model {}", m); + manager.getModelInfo(m).loadWeightVector(); + } + + log.info("Finished loading all models"); + } + + public PredicateArgumentView getSRL(TextAnnotation ta) throws Exception { + log.debug("Input: {}", ta.getText()); + + List predicates; + if (manager.getSRLType() == SRLType.Verb) + predicates = manager.getHeuristicPredicateDetector().getPredicates(ta); + else + predicates = manager.getLearnedPredicateDetector().getPredicates(ta); + + if (predicates.isEmpty()) + return null; + ILPSolverFactory s = new ILPSolverFactory(properties.getILPSolverType(false)); + SRLILPInference inference = new SRLILPInference(s, manager, predicates); + + return inference.getOutputView(); + } + + @Override + public void addView(TextAnnotation ta) throws AnnotatorException { + // Check if all required views are present + + if (ta.hasView(ViewNames.PARSE_STANFORD) && !ta.hasView(ViewNames.CLAUSES_STANFORD)) + ta.addView(ClauseViewGenerator.STANFORD); + + for (String view : getRequiredViews()) { + if (!ta.hasView(view)) { + throw new AnnotatorException("Missing required view: " + view); + } + } + + try { + View srlView = getSRL(ta); + ta.addView(getViewName(), srlView); + } catch (Exception e) { + e.printStackTrace(); + throw new AnnotatorException(e.getMessage()); + } + } + + @Override + public String getViewName() { + if (manager.getSRLType() == SRLType.Verb) { + return ViewNames.SRL_VERB; + } else if (manager.getSRLType() == SRLType.Nom) + return ViewNames.SRL_NOM; + return null; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/caches/FeatureVectorCacheFile.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/caches/FeatureVectorCacheFile.java new file mode 100644 index 000000000..2b3d4f7e2 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/caches/FeatureVectorCacheFile.java @@ -0,0 +1,159 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.caches; + +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.sl.core.SLProblem; +import edu.illinois.cs.cogcomp.sl.util.IFeatureVector; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassLabel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.Iterator; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class FeatureVectorCacheFile implements Closeable, + Iterator> { + + private final static Logger log = LoggerFactory.getLogger(FeatureVectorCacheFile.class); + + private BufferedWriter writer; + private BufferedReader reader; + private String file; + private Models model; + private SRLManager manager; + + private String nextLine = null; + + public FeatureVectorCacheFile(String file, Models model, SRLManager manager) throws IOException { + this.file = file; + this.model = model; + this.manager = manager; + } + + private void openWriter(String file) throws IOException { + BufferedOutputStream stream = + new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(file))); + + writer = new BufferedWriter(new OutputStreamWriter(stream)); + } + + public synchronized void put(String lemma, int label, IFeatureVector features) throws Exception { + + if (writer == null) { + openWriter(file); + } + StringBuilder sb = new StringBuilder(); + sb.append(lemma).append("\t").append(label).append("\t"); + int[] idx = features.getIndices(); + float[] value = features.getValues(); + for (int i = 0; i < idx.length; i++) { + sb.append(idx[i]).append(":").append(value[i]).append(" "); + } + + writer.write(sb.toString().trim()); + writer.newLine(); + } + + public void close() { + try { + if (writer != null) + writer.close(); + + if (reader != null) + reader.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public synchronized Pair next() { + try { + assert reader != null; + + if (nextLine == null) + hasNext(); + + String[] parts = nextLine.split("\t"); + String lemma = parts[0].trim(); + int label = Integer.parseInt(parts[1]); + + String features = parts[2]; + + SRLMulticlassInstance x = new SRLMulticlassInstance(model, lemma, features); + SRLMulticlassLabel y = new SRLMulticlassLabel(label, model, manager); + + return new Pair<>(x, y); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public void openReader() throws IOException { + GZIPInputStream zipin = new GZIPInputStream(new FileInputStream(file)); + reader = new BufferedReader(new InputStreamReader(zipin)); + } + + @Override + public boolean hasNext() { + try { + if (reader == null) + openReader(); + + nextLine = reader.readLine(); + + if (nextLine == null) + return false; + + nextLine = nextLine.trim(); + + return true; + + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + @Override + public void remove() {} + + public SLProblem getStructuredProblem() { + return getStructuredProblem(-1); + } + + public SLProblem getStructuredProblem(int sizeLimit) { + int count = 0; + SLProblem problem = new SLProblem(); + + log.info("Creating structured problem"); + while (hasNext()) { + Pair pair = next(); + problem.instanceList.add(pair.getFirst()); + problem.goldStructureList.add(pair.getSecond()); + + count++; + if (sizeLimit >= 0 && count >= sizeLimit) + break; + + if (count % 10000 == 0) { + log.info("{} examples loaded", count); + } + } + + log.info("{} examples loaded. Finished creating structured problem", count); + + return problem; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/config/LearnerConfigurator.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/config/LearnerConfigurator.java new file mode 100644 index 000000000..a083b33ae --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/config/LearnerConfigurator.java @@ -0,0 +1,71 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.config; + +import edu.illinois.cs.cogcomp.core.utilities.configuration.Configurator; +import edu.illinois.cs.cogcomp.core.utilities.configuration.Property; +import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager; + +/** + * A configuration helper to allow centralization of config options in dependencies with clear + * default settings (only values that override defaults need to be specified). + * + * Created by mssammon on 12/21/15. + */ +public class LearnerConfigurator extends Configurator { + + + /** + * Available learning models: {L2LossSSVM, StructuredPerceptron} + */ + public static final Property LEARNING_MODEL = new Property("LEARNING_MODEL", "L2LossSSVM"); + + /** + * Available solver types: {DCDSolver, ParallelDCDSolver, DEMIParallelDCDSolver} + */ + public static final Property L2_LOS_SSVM_SOLVER_TYPE = new Property("L2_LOSS_SSVM_SOLVER_TYPE", + "ParallelDCDSolver"); + + + public static final Property NUM_THREADS = new Property("NUMBER_OF_THREADS", "8"); + + /** + * Regularization parameter + */ + public static final Property REGULARIZATION = new Property("C_FOR_STRUCTURE", "1.0"); + + + /** + * Mini-bat ch for 'warm' start + */ + public static final Property USE_TRAINMINI = new Property("TRAINMINI", TRUE); + public static final Property TRAINMINI_SIZE = new Property("TRAINMINI_SIZE", "10000"); + + + /** + * Suppress optimatility check + */ + public static final Property CHECK_INFERENCE_OPT = new Property("CHECK_INFERENCE_OPT", FALSE); + + /** + * Number of training rounds + */ + + public static final Property MAX_NUM_ITER = new Property("MAX_NUM_ITER", "100"); + + + + @Override + public ResourceManager getDefaultConfig() { + Property[] properties = + {LEARNING_MODEL, L2_LOS_SSVM_SOLVER_TYPE, NUM_THREADS, REGULARIZATION, + USE_TRAINMINI, TRAINMINI_SIZE, CHECK_INFERENCE_OPT, MAX_NUM_ITER}; + + return new ResourceManager(generateProperties(properties)); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/config/SrlConfigurator.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/config/SrlConfigurator.java new file mode 100644 index 000000000..d574ea3ff --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/config/SrlConfigurator.java @@ -0,0 +1,96 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.config; + +import edu.illinois.cs.cogcomp.annotation.AnnotatorConfigurator; +import edu.illinois.cs.cogcomp.core.utilities.configuration.Configurator; +import edu.illinois.cs.cogcomp.core.utilities.configuration.Property; +import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager; +import edu.illinois.cs.cogcomp.srl.core.SRLType; + +/** + * A configuration helper to allow centralization of config options in dependencies with clear + * default settings (only values that override defaults need to be specified). + * + * NOTE: The parameters for Penn Treebank, Nombank, and Propbank are set to directories on CCG + * servers, so must be overridden on other machines for uses of SRL that require these corpora + * (presumably, for training only). + * + * Created by mssammon on 12/21/15. + */ +public class SrlConfigurator extends AnnotatorConfigurator { + + + public static final Property USE_CURATOR = new Property("UseCurator", FALSE); + public static final Property DEFAULT_PARSER = new Property("DefaultParser", "Stanford"); + + /** + * Num of threads for feat. ext. + */ + public static final Property NUM_FEX_THREADS = new Property("NumFeatExtThreads", "10"); + + /** + * The ILP solver to use for the joint inference Options are: Gurobi, OJAlgo + */ + public static final Property ILP_SOLVER = new Property("ILPSolver", "OJAlgo"); + /** + * Training corpora directories + */ + // This is the directory of the merged (mrg) WSJ files + public static final Property PENNTB_HOME = new Property("PennTreebankHome", + "/shared/corpora/corporaWeb/treebanks/eng/pennTreebank/treebank-3/parsed/mrg/wsj/"); + public static final Property PROPBANK_PARSER = new Property("PropbankHome", + "/shared/corpora/corporaWeb/treebanks/eng/propbank_1/data"); + public static final Property NOMBANK_HOME = new Property("NombankHome", + "/shared/corpora/corporaWeb/treebanks/eng/nombank/"); + + + /** + * The directory of the sentence and pre-extracted features database (~5G of space required) Not + * used during test/working with pre-trained models + */ + public static final Property CACHE_DIR = new Property("CacheDirectory", "cache"); + + public static final Property MODEL_DIR = new Property("ModelsDirectory", "models"); + + + /** + * Directory to output gold and predicted files for manual comparison + */ + public static final Property OUTPUT_DIR = new Property("OutputDirectory", "srl-out"); + + + /** + * whether to instantiate a preprocessing pipeline to provide needed inputs to standalone SRL + */ + public static final Property INSTANTIATE_PREPROCESSOR = new Property("instantiatePreprocessor", + FALSE); + + /** + * whether to use lazy initialization to defer loading of models etc until actually needed + */ + public static final Property LAZILY_INITIALIZE = new Property( + AnnotatorConfigurator.IS_LAZILY_INITIALIZED.key, FALSE); + + + /** + * SRL models to load + */ + public static final Property SRL_TYPE = new Property("srlType", SRLType.Verb.name()); + + + @Override + public ResourceManager getDefaultConfig() { + Property[] properties = + {USE_CURATOR, DEFAULT_PARSER, NUM_FEX_THREADS, PENNTB_HOME, PROPBANK_PARSER, + NOMBANK_HOME, CACHE_DIR, MODEL_DIR, OUTPUT_DIR, ILP_SOLVER, + INSTANTIATE_PREPROCESSOR, LAZILY_INITIALIZE, SRL_TYPE}; + return (new LearnerConfigurator().getConfig(new ResourceManager( + generateProperties(properties)))); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/AbstractPredicateDetector.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/AbstractPredicateDetector.java new file mode 100644 index 000000000..191a419a8 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/AbstractPredicateDetector.java @@ -0,0 +1,57 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.core; + +import edu.illinois.cs.cogcomp.core.datastructures.Option; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; + +import java.util.ArrayList; +import java.util.List; + +/** + * A skeleton for a predicate detector. This could be implemented either by heuristics (both verb + * and nominalization SRL systems come with heuristics) and also a learned predicate detector. + * + * @author Vivek Srikumar + * + */ +public abstract class AbstractPredicateDetector { + + private final SRLManager manager; + + public AbstractPredicateDetector(SRLManager manager) { + this.manager = manager; + } + + public boolean debug = false; + + public abstract Option getLemma(TextAnnotation ta, int tokenId) throws Exception; + + public List getPredicates(TextAnnotation ta) throws Exception { + List list = new ArrayList<>(); + + for (int i = 0; i < ta.size(); i++) { + Option opt = getLemma(ta, i); + + if (opt.isPresent()) { + Constituent c = new Constituent("", "", ta, i, i + 1); + c.addAttribute(PredicateArgumentView.LemmaIdentifier, opt.get()); + list.add(c); + } + } + + return list; + } + + public SRLManager getManager() { + return manager; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/ArgumentCandidateGenerator.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/ArgumentCandidateGenerator.java new file mode 100644 index 000000000..3c67110a5 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/ArgumentCandidateGenerator.java @@ -0,0 +1,45 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.core; + + +import java.util.List; + +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Relation; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; + +/** + * The base class for argument candidate generators. + * + * @author Vivek Srikumar + * + */ +public abstract class ArgumentCandidateGenerator { + + protected final SRLManager manager; + + public ArgumentCandidateGenerator(SRLManager manager) { + this.manager = manager; + } + + public abstract String getCandidateViewName(); + + public abstract List generateCandidates(Constituent predicate); + + protected Constituent getNewConstituent(TextAnnotation ta, Constituent predicateClone, + int start, int end) { + + Constituent newConstituent = + new Constituent("", 1.0, getCandidateViewName(), ta, start, end); + + new Relation("ChildOf", predicateClone, newConstituent, 1.0); + return newConstituent; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/ArgumentIdentifier.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/ArgumentIdentifier.java new file mode 100644 index 000000000..f17e6b7b0 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/ArgumentIdentifier.java @@ -0,0 +1,78 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.core; + +import edu.illinois.cs.cogcomp.core.transformers.Predicate; +import edu.illinois.cs.cogcomp.sl.util.WeightVector; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassLabel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The argument identifier, that uses the scores from the learned model and scales it for high + * recall. The scaling function has two parameters (A,B) and the scaled score is A * + * classifier-score + B. The scaled decision is true if this is positive. + * + * @author Vivek Srikumar + * + */ +@SuppressWarnings("serial") +public class ArgumentIdentifier extends Predicate { + + private final static Logger log = LoggerFactory.getLogger(ArgumentIdentifier.class); + + private final double A; + private final double B; + + private final SRLManager manager; + + public ArgumentIdentifier(double A, double B, SRLManager manager) { + this.A = A; + this.B = B; + this.manager = manager; + + } + + public boolean getIdentifierScaledDecision(SRLMulticlassInstance x) { + try { + return getIdentifierScaledScore(x) >= 0; + } catch (Exception e) { + log.error("Unable to get identifier decision", e); + throw new RuntimeException(e); + } + } + + public double getIdentifierScaledScore(SRLMulticlassInstance x) throws Exception { + return scaleIdentifierScore(getIdentifierRawScore(x)); + } + + public double scaleIdentifierScore(double identifierRawScore) { + double score = A * identifierRawScore + B; + log.debug("Scaled score = {}, raw score = {}", score, identifierRawScore); + return score; + } + + public double getIdentifierRawScore(SRLMulticlassInstance x) throws Exception { + log.debug("Classifying {}", x); + WeightVector w = manager.getModelInfo(Models.Identifier).getWeights(); + + SRLMulticlassLabel y1 = new SRLMulticlassLabel(1, Models.Identifier, manager); + SRLMulticlassLabel y0 = new SRLMulticlassLabel(0, Models.Identifier, manager); + double score1 = + w.dotProduct(x.getCachedFeatureVector(Models.Identifier), + 1 * manager.getModelInfo(Models.Identifier).getLexicon().size()); + double score2 = w.dotProduct(x.getCachedFeatureVector(Models.Identifier)); + return (double) (score1 - score2); + } + + @Override + public Boolean transform(SRLMulticlassInstance input) { + return this.getIdentifierScaledDecision(input); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/LearnedPredicateDetector.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/LearnedPredicateDetector.java new file mode 100644 index 000000000..ffc035da1 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/LearnedPredicateDetector.java @@ -0,0 +1,92 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.core; + +import edu.illinois.cs.cogcomp.core.datastructures.Option; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.View; +import edu.illinois.cs.cogcomp.edison.features.Feature; +import edu.illinois.cs.cogcomp.sl.util.WeightVector; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassLabel; + +import java.util.List; +import java.util.Set; + +public class LearnedPredicateDetector extends AbstractPredicateDetector { + + private AbstractPredicateDetector heuristic; + private final String heuristicPredicateView; + private WeightVector w; + + public LearnedPredicateDetector(SRLManager manager) throws Exception { + super(manager); + heuristic = manager.getHeuristicPredicateDetector(); + heuristicPredicateView = "HeuristicPredicateView:" + manager.getSRLType().name(); + w = manager.getModelInfo(Models.Predicate).getWeights(); + } + + @Override + public Option getLemma(TextAnnotation ta, int tokenId) throws Exception { + if (!ta.hasView(heuristicPredicateView)) { + addHeuristicPredicateView(ta); + } + + View view = ta.getView(heuristicPredicateView); + + List constituentsCoveringToken = view.getConstituentsCoveringToken(tokenId); + if (constituentsCoveringToken.size() == 0) + return Option.empty(); + + Constituent c = constituentsCoveringToken.get(0); + + String lemma = c.getAttribute(PredicateArgumentView.LemmaIdentifier); + + SRLManager manager = getManager(); + + boolean isPredicate; + if (manager.getSRLType() == SRLType.Verb && lemma.equals("be")) { + isPredicate = true; + } else { + SRLMulticlassInstance x = new SRLMulticlassInstance(c, c, manager); + + Set features = manager.getModelInfo(Models.Predicate).fex.getFeatures(c); + x.cacheFeatureVector(Models.Predicate, features); + + SRLMulticlassLabel y0 = new SRLMulticlassLabel(0, Models.Predicate, manager); + SRLMulticlassLabel y1 = new SRLMulticlassLabel(1, Models.Predicate, manager); + double score1 = + w.dotProduct(x.getCachedFeatureVector(Models.Predicate), + manager.getModelInfo(Models.Predicate).getLexicon().size()); + double score2 = w.dotProduct(x.getCachedFeatureVector(Models.Predicate)); + + double score = score1 - score2; + + if (debug) { + System.out.println("Score = " + score); + } + isPredicate = score >= 0; + } + + if (isPredicate) { + return new Option<>(lemma); + } else + return Option.empty(); + + } + + private void addHeuristicPredicateView(TextAnnotation ta) throws Exception { + View view = new View(heuristicPredicateView, "", ta, 1.0); + List predicates = heuristic.getPredicates(ta); + for (Constituent c : predicates) + view.addConstituent(c); + ta.addView(heuristicPredicateView, view); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/ModelInfo.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/ModelInfo.java new file mode 100644 index 000000000..1ec72495e --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/ModelInfo.java @@ -0,0 +1,140 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.core; + +import edu.illinois.cs.cogcomp.core.datastructures.Lexicon; +import edu.illinois.cs.cogcomp.core.io.IOUtils; +import edu.illinois.cs.cogcomp.edison.features.FeatureExtractor; +import edu.illinois.cs.cogcomp.edison.features.manifest.FeatureManifest; +import edu.illinois.cs.cogcomp.sl.util.WeightVector; +import edu.illinois.cs.cogcomp.srl.utilities.WeightVectorUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.List; + +/** + * All information about a model: The feature extractor and the weight vector. At training time, + * this also gives cached features. + * + * @author Vivek Srikumar + * + */ +public class ModelInfo { + private final static Logger log = LoggerFactory.getLogger(ModelInfo.class); + + public final FeatureManifest featureManifest; + public final FeatureExtractor fex; + + private WeightVector w; + + private final SRLManager manager; + + private Models model; + + private Lexicon lexicon; + + public ModelInfo(SRLManager manager, Models m) throws Exception { + this.manager = manager; + this.model = m; + + String file = "features/" + manager.getSRLType() + "." + m + ".fex"; + log.info("Loading feature extractor for {} from {}", m, file); + featureManifest = new FeatureManifest(file); + featureManifest.useCompressedName(); + featureManifest.setVariable("*default-parser*", manager.defaultParser); + + fex = featureManifest.createFex(); + } + + public WeightVector getWeights() { + loadWeightVector(); + return w; + } + + public void loadWeightVector() { + if (w != null) { + log.debug("Weight vector already loaded!"); + return; + } + + synchronized (manager) { + if (w == null) { + String modelFile = manager.getModelFileName(model); + long start = System.currentTimeMillis(); + + if (!IOUtils.exists(modelFile)) { + log.info("Loading weight vector for {} from {} in classpath", model, modelFile); + w = WeightVectorUtils.loadWeightVectorFromClassPath(modelFile); + } else { + log.info("Loading weight vector for {} from {}", model, modelFile); + w = WeightVectorUtils.load(modelFile); + } + long end = System.currentTimeMillis(); + log.info("Finished loading {} weight vector. Took {} ms", model, (end - start)); + } + } + } + + /** + * This function checks if the lexicon file exists. If so, it loads the file. Otherwise, it + * creates a new lexicon. + */ + private Lexicon loadLexicon() throws IOException { + Lexicon lexicon; + String lexiconFile = manager.getLexiconFileName(model); + URL url = null; + + try { + if (!IOUtils.exists(lexiconFile)) { + List list = IOUtils.lsResources(SRLManager.class, lexiconFile); + if (list.size() > 0) + url = list.get(0); + } else { + url = new File(lexiconFile).toURI().toURL(); + } + } catch (Exception ex) { + throw new RuntimeException(ex); + } + + if (url == null) { + log.info("Lexicon file {} missing. Creating new lexicon.", lexiconFile); + lexicon = new Lexicon(true, false); + } else { + log.info("Lexicon file {} found.", lexiconFile); + long start = System.currentTimeMillis(); + lexicon = new Lexicon(url.openStream()); + long end = System.currentTimeMillis(); + + log.info("Finished loading {} lexicon. Took {} ms", model, (end - start)); + } + + return lexicon; + + } + + public Lexicon getLexicon() { + if (this.lexicon == null) { + synchronized (this) { + if (this.lexicon == null) { + try { + this.lexicon = loadLexicon(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } + + return lexicon; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/Models.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/Models.java new file mode 100644 index 000000000..bdc2556ff --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/Models.java @@ -0,0 +1,12 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.core; + +public enum Models { + Classifier, Identifier, Sense, Predicate +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/SRLExampleGenerator.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/SRLExampleGenerator.java new file mode 100644 index 000000000..ed9d00b54 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/SRLExampleGenerator.java @@ -0,0 +1,128 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.core; + +import edu.illinois.cs.cogcomp.core.datastructures.IntPair; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Relation; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.srl.learn.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SRLExampleGenerator { + private SRLManager manager; + + SRLExampleGenerator(SRLManager manager) { + this.manager = manager; + } + + public Pair getExamples(TextAnnotation ta) + throws Exception { + List predicates = new ArrayList<>(); + List structures = new ArrayList<>(); + + if (ta.hasView(manager.getGoldViewName())) + getTreebankExamples(ta, predicates, structures); + else + getExamples(ta, predicates); + + SRLSentenceInstance sx = new SRLSentenceInstance(predicates); + SRLSentenceStructure sy = new SRLSentenceStructure(sx, structures); + + return new Pair<>(sx, sy); + } + + /** + * Generates SRL examples using the predicate detector to identify predicates + */ + private void getExamples(TextAnnotation ta, List predicates) + throws Exception { + AbstractPredicateDetector predicateDetector = manager.getLearnedPredicateDetector(); + + for (Constituent predicate : predicateDetector.getPredicates(ta)) { + if (!predicate.hasAttribute(PredicateArgumentView.LemmaIdentifier)) { + System.out.println(ta); + System.out.println(predicate + " has no lemma!"); + assert false; + } + + SRLPredicateInstance x = new SRLPredicateInstance(predicate, manager); + predicates.add(x); + } + } + + private void getTreebankExamples(TextAnnotation ta, List predicates, + List structures) { + PredicateArgumentView pav = (PredicateArgumentView) ta.getView(manager.getGoldViewName()); + + for (Constituent predicate : pav.getPredicates()) { + if (!predicate.hasAttribute(PredicateArgumentView.LemmaIdentifier)) { + System.out.println(ta); + System.out.println(pav); + System.out.println(predicate + " has no lemma!"); + assert false; + } + + SRLPredicateInstance x = new SRLPredicateInstance(predicate, manager); + Map args = getGoldArgumentSpanLabels(predicate); + int[] argLabels = new int[x.getCandidateInstances().size()]; + + int id = 0; + for (SRLMulticlassInstance c : x.getCandidateInstances()) { + if (args.containsKey(c.getSpan())) { + String label = args.get(c.getSpan()); + + // A hack to deal with invalid Propbank/Nombank data. + if (label.startsWith("AM") && !manager.getAllArguments().contains(label)) { + System.out.println(ta); + System.out.println(pav); + + if (label.equals("AM-TM")) + label = "AM-TMP"; + else if (manager.getSRLType() == SRLType.Nom) { + // the two errors in NomBank are both AM-TMP + label = "AM-TMP"; + } else { + System.out.println("Replacing " + label + " with AM-LOC"); + label = "AM-LOC"; + } + } + + argLabels[id] = manager.getArgumentId(label); + } else + argLabels[id] = manager.getArgumentId(SRLManager.NULL_LABEL); + id++; + } + + int sense = + manager.getSenseId(predicate.getAttribute(PredicateArgumentView.SenseIdentifer)); + SRLPredicateStructure y = new SRLPredicateStructure(x, argLabels, sense, manager); + + predicates.add(x); + structures.add(y); + } + } + + private Map getGoldArgumentSpanLabels(Constituent predicate) { + List args = predicate.getOutgoingRelations(); + + Map argSpans = new HashMap<>(); + for (Relation r : args) { + Constituent arg = r.getTarget(); + argSpans.put(new IntPair(arg.getStartSpan(), arg.getEndSpan()), r.getRelationName()); + } + return argSpans; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/SRLManager.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/SRLManager.java new file mode 100644 index 000000000..780155fb7 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/SRLManager.java @@ -0,0 +1,512 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.core; + +import edu.illinois.cs.cogcomp.core.algorithms.Sorters; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.io.IOUtils; +import edu.illinois.cs.cogcomp.core.math.MathUtilities; +import edu.illinois.cs.cogcomp.edison.features.Feature; +import edu.illinois.cs.cogcomp.edison.features.manifest.FeatureManifest; +import edu.illinois.cs.cogcomp.infer.ilp.ILPSolverFactory; +import edu.illinois.cs.cogcomp.sl.util.WeightVector; +import edu.illinois.cs.cogcomp.srl.Constants; +import edu.illinois.cs.cogcomp.srl.SRLProperties; +import edu.illinois.cs.cogcomp.srl.data.LegalArguments; +import edu.illinois.cs.cogcomp.srl.features.FeatureGenerators; +import edu.illinois.cs.cogcomp.srl.features.ProjectedPath; +import edu.illinois.cs.cogcomp.srl.inference.SRLConstraints; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPInference; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassInstance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.net.URL; +import java.util.*; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +/** + * Manages the lexicon, models and feature extractors for a verb or nom SRL. + * + * @author Vivek Srikumar + * + */ +public abstract class SRLManager { + + private final static Logger log = LoggerFactory.getLogger(SRLManager.class); + + /** + * Indicates the 'null' label + */ + public final static String NULL_LABEL = ""; + + /** + * A flag that indicates that the manager is in training mode, which prompts extensive caching. + */ + public final boolean trainingMode; + + /** + * This class works with a parse trees from this parser, which controls both candidate + * extraction and feature generation. + */ + public final String defaultParser; + + private final HashMap argToId, senseToId; + + private final Set allArgumentsSet; + + private final LegalArguments knownLegalArguments; + + private final SRLProperties properties = SRLProperties.getInstance(); + + private ArgumentIdentifier identifier; + + private final Set constraints = new HashSet<>(); + + private final Map modelInfo; + + public final SRLExampleGenerator exampleGenerator; + + protected SRLManager(boolean trainingMode, String defaultParser) throws Exception { + this.trainingMode = trainingMode; + this.defaultParser = defaultParser; + + initializeFeatureManifest(defaultParser); + + allArgumentsSet = + Collections.unmodifiableSet(new TreeSet<>(Arrays.asList(getArgumentLabels()))); + + senseToId = getLabelIdMap(getSenseLabels()); + argToId = getLabelIdMap(getArgumentLabels()); + this.knownLegalArguments = new LegalArguments(getSRLType() + ".legal.arguments"); + + log.info("{} Arguments: " + Sorters.sortSet(argToId.keySet()), argToId.size()); + log.info("{} senses: " + Sorters.sortSet(senseToId.keySet()), senseToId.size()); + + modelInfo = new HashMap<>(); + + initializeModelInfo(); + + initializeConstraints(); + + exampleGenerator = new SRLExampleGenerator(this); + } + + /** + * Load all the feature extractors + * + * @throws Exception + */ + private void initializeModelInfo() throws Exception { + for (Models m : Models.values()) { + ModelInfo info = new ModelInfo(this, m); + modelInfo.put(m, info); + } + } + + /** + * Load all the constraints. + * + * @throws Exception + */ + private void initializeConstraints() throws Exception { + String file = "constraints/" + getSRLType() + ".constraints"; + log.info("Adding all constraints specified in {}", file); + InputStream in; + if (IOUtils.exists(file)) { + in = new FileInputStream(new File(file)); + } else { + List ls = IOUtils.lsResources(SRLManager.class, file); + in = ls.get(0).openStream(); + } + Scanner scanner = new Scanner(in); + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + + int semiColon = line.indexOf(';'); + if (semiColon >= 0) + line = line.substring(0, semiColon); + + line = line.trim(); + if (line.length() == 0) + continue; + try { + SRLConstraints constraint = SRLConstraints.valueOf(line); + log.info("Including constraint {}", constraint); + + this.addConstraint(constraint); + } catch (Exception e) { + log.error("Error with constraint {}", line); + throw e; + } + } + + scanner.close(); + in.close(); + } + + protected HashMap getLabelIdMap(String[] strings) { + + HashMap label2Id = new HashMap<>(); + for (int i = 0; i < strings.length; i++) { + label2Id.put(strings[i], i); + } + + return label2Id; + } + + private void initializeFeatureManifest(String defaultParser) { + Feature.setUseAscii(); + Feature.setKeepString(); + + FeatureManifest.setFeatureExtractor("hyphen-argument-feature", + FeatureGenerators.hyphenTagFeature); + + // These three are from Surdeanu etal. + FeatureManifest.setTransformer("parse-left-sibling", + FeatureGenerators.getParseLeftSibling(defaultParser)); + FeatureManifest.setTransformer("parse-right-sibling", + FeatureGenerators.getParseRightSibling(defaultParser)); + FeatureManifest.setFeatureExtractor("pp-features", + FeatureGenerators.ppFeatures(defaultParser)); + + // Introduced in Toutanova etal. + FeatureManifest.setFeatureExtractor("projected-path", new ProjectedPath(defaultParser)); + } + + public Set getAllArguments() { + return allArgumentsSet; + } + + public void addConstraint(SRLConstraints c) { + this.constraints.add(c); + } + + public Set getConstraints() { + return Collections.unmodifiableSet(this.constraints); + } + + public int getNumLabels(Models type) { + if (type == Models.Identifier || type == Models.Predicate) + return 2; + else if (type == Models.Classifier) + return getNumArguments(); + else + return getNumSenses(); + } + + public int getArgumentId(String label) { + // XXX: Not sure why I'm doing this. Some misguided sense of continuity + // to the old data format, I think. I'm sure I will pay hell for this + // sometime in the future with hours of debugging! + label = label.replace("Support", "SUP"); + + if (argToId.containsKey(label)) + return argToId.get(label); + else { + log.debug(label + " is not a valid argument. Expecting one of " + this.argToId.keySet() + + ", replacing with " + NULL_LABEL); + return argToId.get(NULL_LABEL); + } + } + + public int getSenseId(String label) { + // XXX: special case for Propbank. Not clear what this label means, but + // it occurs occasionally in the data and we don't really want to + // predict this + if (label.equals("XX")) + return senseToId.get("01"); + else if (senseToId.containsKey(label)) + return senseToId.get(label); + else + throw new NullPointerException(label + " not a valid sense"); + } + + public abstract SRLType getSRLType(); + + protected abstract String[] getArgumentLabels(); + + protected abstract String[] getSenseLabels(); + + public abstract Set getCoreArguments(); + + public abstract Set getModifierArguments(); + + protected abstract int getNumArguments(); + + protected abstract int getNumSenses(); + + public abstract String getArgument(int id); + + public abstract String getSense(int id); + + public abstract ArgumentCandidateGenerator getArgumentCandidateGenerator(); + + public abstract AbstractPredicateDetector getHeuristicPredicateDetector(); + + public abstract AbstractPredicateDetector getLearnedPredicateDetector(); + + public abstract int getPruneSize(Models model); + + public ModelInfo getModelInfo(Models model) { + return modelInfo.get(model); + } + + public ArgumentIdentifier getArgumentIdentifier() { + if (this.identifier == null) { + synchronized (this) { + if (this.identifier == null) { + log.info("Loading argument identifier"); + try { + InputStream in; + if (!IOUtils.exists(getIdentifierScaleFile())) { + log.debug("Looking for {} in the classpath", getIdentifierScaleFile()); + List urls = + IOUtils.lsResources(SRLManager.class, getIdentifierScaleFile()); + if (urls.size() == 0) + log.error("Argument identifier scale file not found!"); + URL url = urls.get(0); + + in = url.openStream(); + } else { + log.debug("Looking for {} in the file system", getIdentifierScaleFile()); + in = new FileInputStream(new File(getIdentifierScaleFile())); + } + Pair pair = readIdentifierScale(in); + this.identifier = + new ArgumentIdentifier(pair.getFirst(), pair.getSecond(), this); + + log.info("Finished initializing argument identifier"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + } + return this.identifier; + } + + public void writeIdentifierScale(double A, double B) throws IOException { + + String file = getIdentifierScaleFile(); + + log.info("Writing identifier scaling info to {}", file); + + BufferedOutputStream stream = + new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(file))); + + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(stream)); + + writer.write("IdentifierScale"); + writer.newLine(); + writer.write("" + A); + writer.newLine(); + writer.write("" + B); + + writer.close(); + } + + private Pair readIdentifierScale(InputStream in) throws IOException { + GZIPInputStream zipin = new GZIPInputStream(in); + BufferedReader reader = new BufferedReader(new InputStreamReader(zipin)); + + String line; + + line = reader.readLine().trim(); + if (!line.equals("IdentifierScale")) { + throw new IOException("Invalid identifier scalefile"); + } + + double A = Double.parseDouble(reader.readLine().trim()); + double B = Double.parseDouble(reader.readLine().trim()); + + log.info("Argument identifier scaler (A,B) = ({},{})", A, B); + + zipin.close(); + + return new Pair<>(A, B); + } + + private String getFeatureIdentifier(Models type) { + return getModelInfo(type).featureManifest.getIncludedFeatures().replaceAll("\\s+", "") + .hashCode() + + ""; + } + + public String getGoldViewName() { + return getPredictedViewName() + "_GOLD"; + } + + public abstract String getPredictedViewName(); + + public String getSRLSystemIdentifier() { + if (this.getSRLType() == SRLType.Verb) + return Constants.verbSRLSystemIdentifier; + else + return Constants.nomSRLSystemIdentifier; + } + + /** + * The name of the file that contains the identifer scale information + */ + public String getIdentifierScaleFile() { + String identifier = getFeatureIdentifier(Models.Identifier); + return properties.getModelsDir() + "/" + this.getSRLType() + "." + Models.Identifier.name() + + "." + defaultParser + "." + identifier + ".scale"; + } + + /** + * The name of the file that contains the lexicon + */ + public String getLexiconFileName(Models m) { + return properties.getModelsDir() + "/" + getSRLType() + "." + m + "." + defaultParser + + ".lex"; + } + + /** + * The name of the file that contains the model for the given model type + */ + public String getModelFileName(Models type) { + String identifier = getFeatureIdentifier(type); + return properties.getModelsDir() + "/" + this.getSRLType() + "." + type.name() + "." + + defaultParser + "." + identifier + ".lc"; + } + + /** + * Checks if the given sense id is a valid predicate sense for the given predicate. The sense id + * should be valid according to the function getSenseId. + */ + public boolean isValidSense(String predicate, int senseId) { + assert 0 <= senseId && senseId < getNumSenses(); + return this.getLegalSenses(predicate).contains(getSense(senseId)); + } + + /** + * Checks if the input label is a valid label Id of the specified model type for the input x. + */ + public boolean isValidLabel(SRLMulticlassInstance x, Models type, int label) { + if (type == Models.Identifier || type == Models.Predicate) + return label == 0 || label == 1; + else if (type == Models.Classifier) + return this.getLegalArguments(x.getPredicateLemma()).contains(this.getArgument(label)); + else + return this.getLegalSenses(x.getPredicateLemma()).contains(this.getSense(label)); + } + + /** + * Checks if the argument id specified is the special NULL_LABEL. + */ + public boolean isNullLabel(int argLabelId) { + return NULL_LABEL.equals(this.getArgument(argLabelId)); + } + + /** + * Returns the set of legal arguments for the given lemma. This function uses the frame files to + * get the list of valid core arguments. All the modifiers are treated as legal arguments. In + * addition, all C-args and R-args of legal core/modifier arguments are also considered legal. + * For unknown predicates, all arguments are legal. + */ + public Set getLegalArguments(String lemma) { + if (knownLegalArguments.hasLegalArguments(lemma)) { + Set set = new HashSet<>(); + + set.addAll(Arrays.asList("AM-ADV", "AM-DIS", "AM-LOC", "AM-MNR", "AM-MOD", "AM-NEG", + "AM-TMP")); + + set.addAll(knownLegalArguments.getLegalArguments(lemma)); + + return set; + } else + return getAllArguments(); + } + + /** + * Get the set of valid senses for this predicate using the frame files. For unknown predicates, + * only the sense 01 is allowed. + */ + public Set getLegalSenses(String predicate) { + if (knownLegalArguments.hasLegalSenses(predicate)) { + Set senses = knownLegalArguments.getLegalSenses(predicate); + + // keep only senses that the model knows about + senses.retainAll(this.senseToId.keySet()); + + if (senses.size() > 0) + return senses; + else { + log.error("Unknown predicate {}. Allowing only sense 01", predicate); + } + } + return new HashSet<>(Collections.singletonList("01")); + } + + /** + * Get valid arguments for each sense of a given lemma from the frame files. For unknown + * predicates, only the sense 01 is allowed with all arguments + */ + public Map> getLegalLabelsForSense(String lemma) { + Map> map = new HashMap<>(); + + if (knownLegalArguments.hasLegalSenses(lemma) + && knownLegalArguments.hasLegalArguments(lemma)) { + for (String sense : knownLegalArguments.getLegalSenses(lemma)) { + Set argsForSense = + new HashSet<>(knownLegalArguments.getLegalArgsForSense(lemma, sense)); + argsForSense.add(NULL_LABEL); + + map.put(sense, argsForSense); + } + } else { + map.put("01", getAllArguments()); + } + + return map; + } + + /** + * Scores instance for the different labels allowed for it + */ + public double[] getScores(SRLMulticlassInstance x, Models type, boolean rescoreInvalidLabels) { + int numLabels = this.getNumLabels(type); + double[] scores = new double[numLabels]; + + WeightVector w; + try { + w = this.getModelInfo(type).getWeights(); + assert w != null; + } catch (Exception e) { + log.error("Unable to load weight vector for {}", type, e); + throw new RuntimeException(e); + } + + for (int label = 0; label < numLabels; label++) { + + if (!this.isValidLabel(x, type, label) && rescoreInvalidLabels) { + scores[label] = -50; + } else { + scores[label] = + w.dotProduct(x.getCachedFeatureVector(type), label + * this.getModelInfo(type).getLexicon().size()); + } + } + + scores = MathUtilities.softmax(scores); + + return scores; + + } + + public SRLILPInference getInference(ILPSolverFactory solver, List predicates) + throws Exception { + return new SRLILPInference(solver, this, predicates); + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/SRLType.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/SRLType.java new file mode 100644 index 000000000..2e74f87d3 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/core/SRLType.java @@ -0,0 +1,19 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.core; + +/** + * A list of the types of SRL that are based on the treebank. This is a convenient enumeration that + * can be used to generalize the very similar verb and nominal SRL systems. + * + * @author Vivek Srikumar + * + */ +public enum SRLType { + Verb, Nom +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/Dataset.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/Dataset.java new file mode 100644 index 000000000..fef172045 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/Dataset.java @@ -0,0 +1,22 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.data; + +public enum Dataset { + PTBAll, PTBTrain, PTBDev, PTBTest, PTB0204, PTBTrainDev; + + public static String[] stringValues() { + String[] values = new String[Dataset.values().length]; + Dataset[] datasets = Dataset.values(); + for (int i = 0; i < datasets.length; i++) { + Dataset d = datasets[i]; + values[i] = d.name(); + } + return values; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/FrameData.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/FrameData.java new file mode 100644 index 000000000..dbc1e50c5 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/FrameData.java @@ -0,0 +1,115 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.data; + +import java.util.*; + +/** + * @author Vivek Srikumar + */ +public class FrameData { + private String lemma; + + private static class SenseFrameData { + Map argDescription = new HashMap<>(); + String verbClass = "UNKNOWN"; + String senseName; + List examples = new ArrayList<>(); + } + + private static class ArgumentData { + String description; + Set vnTheta = new HashSet<>(); + } + + private static class Example { + String text; + String name; + Map argDescriptions = new HashMap<>(); + Map argExamples = new HashMap<>(); + } + + private Map senseFrameData; + + public FrameData(String lemma) { + this.lemma = lemma; + senseFrameData = new HashMap<>(); + } + + public void addExample(String sense, String name, String text, + Map argDescriptions, Map argExamples) { + Example ex = new Example(); + ex.name = name; + ex.text = text; + ex.argDescriptions = argDescriptions; + ex.argExamples = argExamples; + this.senseFrameData.get(sense).examples.add(ex); + } + + public String getLemma() { + return lemma; + } + + public void addSense(String sense, String senseName, String verbClass) { + this.senseFrameData.put(sense, new SenseFrameData()); + this.senseFrameData.get(sense).verbClass = verbClass; + this.senseFrameData.get(sense).senseName = senseName; + } + + public Set getSenses() { + return this.senseFrameData.keySet(); + } + + public void addArgument(String sense, String arg) { + assert this.senseFrameData.containsKey(sense); + + senseFrameData.get(sense).argDescription.put(arg, new ArgumentData()); + } + + public Set getArgsForSense(String sense) { + + assert this.senseFrameData.containsKey(sense) : sense + " missing for predicate lemma " + + this.lemma; + return this.senseFrameData.get(sense).argDescription.keySet(); + } + + public void addArgumentDescription(String sense, String arg, String description) { + assert this.senseFrameData.containsKey(sense); + assert this.senseFrameData.get(sense).argDescription.containsKey(arg); + + senseFrameData.get(sense).argDescription.get(arg).description = description; + } + + public String getArgumentDescription(String sense, String arg) { + assert this.senseFrameData.containsKey(sense); + assert this.senseFrameData.get(sense).argDescription.containsKey(arg); + + return senseFrameData.get(sense).argDescription.get(arg).description; + } + + public void addArgumentVNTheta(String sense, String arg, String vnTheta) { + assert this.senseFrameData.containsKey(sense); + assert this.senseFrameData.get(sense).argDescription.containsKey(arg); + + senseFrameData.get(sense).argDescription.get(arg).vnTheta.add(vnTheta); + + } + + public Set getLegalArguments() { + Set l = new HashSet<>(); + for (String s : this.getSenses()) + l.addAll(this.getArgsForSense(s)); + return l; + } + + public String getSenseName(String sense) { + assert this.senseFrameData.containsKey(sense) : sense + " missing for predicate lemma " + + this.lemma; + return this.senseFrameData.get(sense).senseName; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/FramesManager.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/FramesManager.java new file mode 100644 index 000000000..e72839d45 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/FramesManager.java @@ -0,0 +1,283 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.data; + +import edu.illinois.cs.cogcomp.core.io.IOUtils; +import edu.illinois.cs.cogcomp.core.io.LineIO; +import edu.illinois.cs.cogcomp.edison.utilities.EdisonException; +import edu.illinois.cs.cogcomp.srl.SRLProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.EntityResolver; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.*; + +/** + * @author Vivek Srikumar + */ +public class FramesManager { + + private final static Logger log = LoggerFactory.getLogger(FramesManager.class); + + public static String propFramesDir = SRLProperties.getInstance().getPropbankHome() + "/frames"; + public static String nomFramesDir = SRLProperties.getInstance().getNombankHome() + "/frames"; + + private static FramesManager PROP_INSTANCE = null, NOM_INSTANCE = null; + + public static FramesManager getPropbankInstance() { + if (PROP_INSTANCE == null) { + synchronized (log) { + if (PROP_INSTANCE == null) { + PROP_INSTANCE = new FramesManager(propFramesDir); + } + } + } + return PROP_INSTANCE; + } + + public static FramesManager getNombankInstance() { + if (NOM_INSTANCE == null) { + synchronized (log) { + if (NOM_INSTANCE == null) { + + NOM_INSTANCE = new FramesManager(nomFramesDir); + } + } + } + return NOM_INSTANCE; + } + + private static final Set prepositions = new HashSet<>(Arrays.asList(new String[] { + "about", "above", "across", "after", "against", "along", "among", "around", "as", "at", + "before", "behind", "beneath", "beside", "between", "by", "down", "during", "for", + "from", "in", "inside", "into", "like", "of", "off", "on", "onto", "over", "round", + "through", "to", "towards", "with"})); + + public static final String UNKNOWN_VERB_CLASS = "UNKNOWN"; + + public FramesManager(String dir) { + log.info("Loading frames from {}", dir); + + try { + readFrameData(dir); + } catch (Exception e) { + log.error("Unable to load frames from {}", dir); + throw new RuntimeException(e); + } + } + + private void readFrameData(String dir) throws Exception { + frameData = new HashMap<>(); + + File framesetFile = new File(dir + File.separator + "frameset.dtd"); + + if (!framesetFile.exists()) + throw new EdisonException("Cannot read frames. 'frameset.dtd' not found in " + dir); + + final URL dtd = framesetFile.toURI().toURL(); + + for (File file : new File(dir).listFiles()) { + + if (!file.getName().endsWith("xml")) + continue; + URL url = file.toURI().toURL(); + String fileName = file.getName(); + + // A hack to deal with percent-sign in nombank. There is another + // file called perc-sign that will fill this void. + if (fileName.contains("percent-sign.xml")) + continue; + + // I HATE IT THAT JAVA XML IS SO PAINFUL! ALL THIS JUNK TO READ A + // BUNCH OF FILES!!!! + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setNamespaceAware(false); + dbf.setValidating(false); + + DocumentBuilder db = dbf.newDocumentBuilder(); + + db.setEntityResolver(new EntityResolver() { + public InputSource resolveEntity(String publicId, String systemId) + throws SAXException, IOException { + return new InputSource(dtd.openStream()); + } + }); + + Document doc = db.parse(url.openStream()); + + NodeList predicateElements = doc.getElementsByTagName("predicate"); + + for (int i = 0; i < predicateElements.getLength(); i++) { + String lemma = IOUtils.stripFileExtension(fileName); + + FrameData fData = new FrameData(lemma); + frameData.put(lemma, fData); + + NodeList roleSets = doc.getElementsByTagName("roleset"); + addRoleSets(fileName, lemma, fData, roleSets); + } + } + } + + private void addRoleSets(String file, String lemma, FrameData fData, NodeList roleSets) { + for (int i = 0; i < roleSets.getLength(); i++) { + Element roleSet = (Element) roleSets.item(i); + + String sense = roleSet.getAttribute("id"); + + String senseName = roleSet.getAttribute("name"); + + // WTF frame makers? + if (sense.equals("lionise.01")) + sense = "lionize.01"; + + if (sense.equals("oneslashonezeroth.01")) + sense = "1-slash-10th.01"; + + assert sense.startsWith(IOUtils.stripFileExtension(file)) || sense.startsWith(lemma) : lemma + + "\t" + sense; + + String verbClass; + if (roleSet.hasAttribute("vncls")) { + verbClass = roleSet.getAttribute("vncls"); + if (verbClass.equals("-") || verbClass.length() == 0) + verbClass = UNKNOWN_VERB_CLASS; + } else + verbClass = UNKNOWN_VERB_CLASS; + + sense = sense.replaceAll(lemma + ".", ""); + + fData.addSense(sense, senseName, verbClass); + + NodeList roles = roleSet.getElementsByTagName("role"); + + for (int j = 0; j < roles.getLength(); j++) { + Element role = (Element) roles.item(j); + + String label = role.getAttribute("n"); + if (role.hasAttribute("f")) + label += "-" + role.getAttribute("f"); + String argLabel = "A" + label; + fData.addArgument(sense, argLabel); + + if (role.hasAttribute("descr")) { + String descr = role.getAttribute("descr"); + fData.addArgumentDescription(sense, argLabel, descr); + } + + NodeList elementsByTagName = role.getElementsByTagName("vnrole"); + + for (int roleId = 0; roleId < elementsByTagName.getLength(); roleId++) { + String vntheta = + ((Element) (elementsByTagName.item(roleId))).getAttribute("vntheta"); + fData.addArgumentVNTheta(sense, argLabel, vntheta); + } + + } // end of arguments + addExamples(fData, roleSet, sense); + } + } + + private void addExamples(FrameData fData, Element roleSet, String sense) { + NodeList examples = roleSet.getElementsByTagName("example"); + + for (int exampleId = 0; exampleId < examples.getLength(); exampleId++) { + Element example = (Element) examples.item(exampleId); + + String name = ""; + if (example.hasAttribute("name")) + name = example.getAttribute("name"); + + String text = example.getElementsByTagName("text").item(0).getTextContent(); + + NodeList args = example.getElementsByTagName("arg"); + + Map argDescriptions = new HashMap<>(); + Map argExamples = new HashMap<>(); + + for (int argId = 0; argId < args.getLength(); argId++) { + Element arg = (Element) args.item(argId); + + String n = ""; + if (arg.hasAttribute("n")) + n = "A" + arg.getAttribute("n"); + + if (n.length() <= 1 || n.toUpperCase().equals("AM")) + continue; + + if (arg.hasAttribute("f")) { + String f = arg.getAttribute("f").toLowerCase(); + if (prepositions.contains(f)) { + argDescriptions.put(n, f); + argExamples.put(n, arg.getTextContent()); + } + } else { + String t = arg.getTextContent(); + String[] parts = t.split(" "); + if (parts.length > 0) { + t = parts[0].toLowerCase(); + if (prepositions.contains(t)) { + argDescriptions.put(n, t); + argExamples.put(n, arg.getTextContent()); + + } + } + } + } + fData.addExample(sense, name, text, argDescriptions, argExamples); + } + } + + public HashMap frameData; + + public Set getPredicates() { + return frameData.keySet(); + } + + public FrameData getFrame(String lemma) { + return frameData.get(lemma); + } + + /** + * Use this to create the compact files used during inference. NB: You need to have access to + * the Propbank and Nombank frame files + */ + public static void main(String[] args) throws IOException { + List outLines = new ArrayList<>(); + FramesManager manager = new FramesManager(propFramesDir); + // FramesManager manager = new FramesManager(nomFramesDir); + for (String predicate : manager.getPredicates()) { + FrameData frame = manager.getFrame(predicate); + Set senses = frame.getSenses(); + String senseStr = ""; + for (String sense : senses) { + Set argsForSense = frame.getArgsForSense(sense); + if (argsForSense.isEmpty()) + continue; + senseStr += sense + "#"; + for (String arg : argsForSense) + senseStr += arg + ","; + senseStr = senseStr.substring(0, senseStr.length() - 1) + " "; + } + outLines.add(predicate + "\t" + senseStr.trim()); + } + Collections.sort(outLines); + LineIO.write("src/main/resources/Verb.legal.arguments", outLines); + // LineIO.write("src/main/resources/Nom.legal.arguments", outLines); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/LegalArguments.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/LegalArguments.java new file mode 100644 index 000000000..9831f73c9 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/data/LegalArguments.java @@ -0,0 +1,85 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.data; + +import edu.illinois.cs.cogcomp.core.io.IOUtils; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URL; +import java.util.*; + +public class LegalArguments { + + private final static Logger log = LoggerFactory.getLogger(LegalArguments.class); + + private final Map> legalArgs = new HashMap<>(); + private final Map> legalArgsForSense = new HashMap<>(); + private final Map> legalSenses = new HashMap<>(); + + public LegalArguments(String file) throws Exception { + List list = IOUtils.lsResources(LegalArguments.class, file); + if (list.size() == 0) { + log.error("Cannot find file " + file + " in the classpath."); + } else { + URL url = list.get(0); + Scanner scanner = new Scanner(url.openStream()); + + log.info("Loading legal arguments from {}", file); + while (scanner.hasNextLine()) { + String line = scanner.nextLine().trim(); + if (line.length() == 0) + continue; + + String[] strings = line.split("\t"); + + String lemma = strings[0].trim(); + + Set set = new HashSet<>(); + Set sensesSet = new HashSet<>(); + + if (strings.length == 2) { + for (String argsForSense : strings[1].split("\\s+")) { + String sense = argsForSense.split("#")[0]; + + List args = Arrays.asList(argsForSense.split("#")[1].split(",")); + legalArgsForSense.put(lemma + "." + sense, new HashSet<>(args)); + set.addAll(args); + sensesSet.add(sense); + } + } + + set.add(SRLManager.NULL_LABEL); + legalArgs.put(lemma, set); + legalSenses.put(lemma, sensesSet); + } + scanner.close(); + } + } + + public boolean hasLegalArguments(String lemma) { + return this.legalArgs.containsKey(lemma); + } + + public Set getLegalArguments(String lemma) { + return this.legalArgs.get(lemma); + } + + public boolean hasLegalSenses(String lemma) { + return this.legalSenses.containsKey(lemma); + } + + public Set getLegalSenses(String lemma) { + return this.legalSenses.get(lemma); + } + + public Set getLegalArgsForSense(String lemma, String sense) { + return this.legalArgsForSense.get(lemma + "." + sense); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/PreExtractRecord.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/PreExtractRecord.java new file mode 100644 index 000000000..9d53f425d --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/PreExtractRecord.java @@ -0,0 +1,23 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.experiment; + +import edu.illinois.cs.cogcomp.sl.util.IFeatureVector; + +class PreExtractRecord { + String lemma; + int label; + IFeatureVector features; + + public PreExtractRecord(String lemma, int label, IFeatureVector features) { + this.lemma = lemma; + this.label = label; + this.features = features; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/PreExtractor.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/PreExtractor.java new file mode 100644 index 000000000..72fc2809a --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/PreExtractor.java @@ -0,0 +1,240 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.experiment; + +import edu.illinois.cs.cogcomp.core.algorithms.ProducerConsumer; +import edu.illinois.cs.cogcomp.core.datastructures.IntPair; +import edu.illinois.cs.cogcomp.core.datastructures.Lexicon; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.utilities.ArrayUtilities; +import edu.illinois.cs.cogcomp.edison.features.Feature; +import edu.illinois.cs.cogcomp.edison.features.FeatureExtractor; +import edu.illinois.cs.cogcomp.edison.utilities.EdisonException; +import edu.illinois.cs.cogcomp.sl.util.FeatureVectorBuffer; +import edu.illinois.cs.cogcomp.sl.util.IFeatureVector; +import edu.illinois.cs.cogcomp.srl.caches.FeatureVectorCacheFile; +import edu.illinois.cs.cogcomp.srl.core.*; +import edu.illinois.cs.cogcomp.srl.learn.*; + +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; + +public class PreExtractor extends ProducerConsumer { + + protected final FeatureVectorCacheFile cacheDB; + + protected final List buffer = new ArrayList<>(); + + protected final SRLManager manager; + private AtomicInteger taCounter = new AtomicInteger(); + private AtomicInteger argCounter = new AtomicInteger(); + private AtomicInteger predicateCounter = new AtomicInteger(); + protected final Models modelToExtract; + + private boolean addNewFeatures; + private final Lexicon lexicon; + + public PreExtractor(SRLManager manager, Iterator data, int numConsumers, + Models modelToExtract, FeatureVectorCacheFile featureCache) { + super(data, numConsumers); + this.modelToExtract = modelToExtract; + this.manager = manager; + this.cacheDB = featureCache; + this.addNewFeatures = true; + + if (modelToExtract == Models.Classifier) + manager.getModelInfo(Models.Identifier).loadWeightVector(); + + lexicon = manager.getModelInfo(modelToExtract).getLexicon(); + } + + @Override + protected void initialize() {} + + @Override + protected boolean prerequisiteCheck(TextAnnotation ta) { + return ta.hasView(manager.getGoldViewName()); + } + + @Override + protected void consume(TextAnnotation ta) { + try { + assert manager.trainingMode; + + if (modelToExtract == Models.Predicate) { + AbstractPredicateDetector detector = manager.getHeuristicPredicateDetector(); + List predicates = detector.getPredicates(ta); + + Set gold = new HashSet<>(); + if (ta.hasView(manager.getGoldViewName())) { + PredicateArgumentView pav = + (PredicateArgumentView) ta.getView(manager.getGoldViewName()); + + for (Constituent pred : pav.getPredicates()) + gold.add(pred.getSpan()); + } + + for (Constituent c : predicates) { + SRLMulticlassInstance predicateInstance = + new SRLMulticlassInstance(c, c, manager); + + int label = gold.contains(c.getSpan()) ? 1 : 0; + SRLMulticlassLabel y = new SRLMulticlassLabel(label, Models.Predicate, manager); + + consumeInstance(predicateInstance, y); + + predicateCounter.incrementAndGet(); + } + } else { + Pair examples = + manager.exampleGenerator.getExamples(ta); + SRLSentenceInstance sentenceInstance = examples.getFirst(); + SRLSentenceStructure sentenceStructure = examples.getSecond(); + + for (int predicateId = 0; predicateId < sentenceInstance.numPredicates(); predicateId++) { + SRLPredicateInstance x = sentenceInstance.predicates.get(predicateId); + SRLPredicateStructure y = sentenceStructure.ys.get(predicateId); + + predicateCounter.incrementAndGet(); + if (modelToExtract == Models.Sense) { + consumeInstance(x.getSenseInstance(), y.getSenseMulticlassLabel()); + } else { + List candidateInstances; + + candidateInstances = x.getCandidateInstances(); + + for (int i = 0; i < candidateInstances.size(); i++) { + SRLMulticlassInstance cand = candidateInstances.get(i); + SRLMulticlassLabel candLabel; + if (modelToExtract == Models.Identifier) + candLabel = y.getIdentifierMulticlassLabel(i); + else + candLabel = y.getClassifierMulticlassLabel(i); + + if (modelToExtract == Models.Classifier) { + ArgumentIdentifier identifier = manager.getArgumentIdentifier(); + + Set idFeats = + manager.getModelInfo(Models.Identifier).fex + .getFeatures(cand.getConstituent()); + cand.cacheFeatureVector(Models.Identifier, idFeats); + + if (!identifier.getIdentifierScaledDecision(cand)) + continue; + } + consumeInstance(cand, candLabel); + argCounter.incrementAndGet(); + } + } + } + } + taCounter.incrementAndGet(); + } catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } + } + + @Override + protected String getStatus() { + String s = + taCounter.get() + " sentences cached, #predicates =" + predicateCounter.get() + + ", #args = " + argCounter.get() + ", "; + + s += "#features = " + manager.getModelInfo(modelToExtract).getLexicon().size(); + + return s; + } + + @Override + protected List process(TextAnnotation ta) { + return Collections.singletonList(ta); + } + + protected void consumeInstance(SRLMulticlassInstance x, SRLMulticlassLabel y) throws Exception { + countFeatures(x); + + synchronized (buffer) { + IFeatureVector fv = x.getCachedFeatureVector(modelToExtract); + assert fv != null; + buffer.add(new PreExtractRecord(x.getPredicateLemma(), y.getLabel(), fv)); + } + + if (buffer.size() > 10000) { + synchronized (buffer) { + if (buffer.size() > 10000) { + for (PreExtractRecord r : buffer) { + cacheDB.put(r.lemma, r.label, r.features); + } + buffer.clear(); + } + } + } + + } + + public void finalize() throws Exception { + for (PreExtractRecord r : buffer) { + cacheDB.put(r.lemma, r.label, r.features); + } + + this.cacheDB.close(); + + try { + super.finalize(); + } catch (Throwable throwable) { + throwable.printStackTrace(); + } + } + + /** + * This is where actual feature extraction is taking place. The features are defined in the + * *.fex files (per SRL type and model) and are read by {@link FeatureExtractor}
+ * NB:For prepSRL we are considering the features of the Governor and Object instead of + * the predicate itself. + * + * @param x The predicate to extract features from. + * @throws EdisonException + */ + public void countFeatures(SRLMulticlassInstance x) throws EdisonException { + ModelInfo modelInfo = manager.getModelInfo(modelToExtract); + + Set feats = modelInfo.fex.getFeatures(x.getConstituent()); + + // This is the only place where a new feature can be added to the lexicon. + List ids = new ArrayList<>(); + List values = new ArrayList<>(); + synchronized (lexicon) { + for (Feature f : feats) { + if (addNewFeatures) { + if (!lexicon.contains(f.getName())) { + lexicon.previewFeature(f.getName()); + } + } else if (!lexicon.contains(f.getName())) { + continue; + } + int featureId = lexicon.lookupId(f.getName()); + lexicon.countFeature(featureId); + + ids.add(featureId); + values.add(f.getValue()); + } + } + FeatureVectorBuffer tmp = + new FeatureVectorBuffer(ArrayUtilities.asIntArray(ids), + ArrayUtilities.asFloatArray(values)); + x.cacheFeatureVector(modelToExtract, tmp.toFeatureVector()); + } + + public void lockLexicon() { + this.addNewFeatures = false; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/PruningPreExtractor.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/PruningPreExtractor.java new file mode 100644 index 000000000..f845e5f63 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/PruningPreExtractor.java @@ -0,0 +1,128 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.experiment; + +import edu.illinois.cs.cogcomp.core.algorithms.ProducerConsumer; +import edu.illinois.cs.cogcomp.core.datastructures.Lexicon; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.sl.util.SparseFeatureVector; +import edu.illinois.cs.cogcomp.srl.caches.FeatureVectorCacheFile; +import edu.illinois.cs.cogcomp.srl.core.ModelInfo; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassLabel; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Creates a second feature cache, where the pruned features are stored. + * + * @author Vivek Srikumar + * + */ +public class PruningPreExtractor extends + ProducerConsumer> { + protected final FeatureVectorCacheFile cache; + + private SRLManager manager; + private Models modelToExtract; + + protected final List buffer = new ArrayList<>(); + + private AtomicInteger counter = new AtomicInteger(); + private Logger log = org.slf4j.LoggerFactory.getLogger(PruningPreExtractor.class); + + public PruningPreExtractor(SRLManager manager, Models modelToExtract, + FeatureVectorCacheFile examples, FeatureVectorCacheFile cache, int nThreads) { + + super(examples, nThreads); + + this.manager = manager; + this.modelToExtract = modelToExtract; + this.cache = cache; + + } + + @Override + protected void initialize() { + + } + + @Override + protected boolean prerequisiteCheck(Pair input) { + return true; + } + + @Override + protected void consume(Pair input) { + + SRLMulticlassInstance x = input.getFirst(); + SRLMulticlassLabel y = input.getSecond(); + + SparseFeatureVector features = + (SparseFeatureVector) x.getCachedFeatureVector(modelToExtract); + + ModelInfo modelInfo = manager.getModelInfo(modelToExtract); + Lexicon lexicon = modelInfo.getLexicon(); + + int threshold = manager.getPruneSize(modelToExtract); + + Pair pair = + lexicon.pruneFeaturesByCount(features.getIndices(), features.getValues(), threshold); + + features = new SparseFeatureVector(pair.getFirst(), pair.getSecond()); + + synchronized (buffer) { + buffer.add(new PreExtractRecord(x.getPredicateLemma(), y.getLabel(), features)); + + } + + if (buffer.size() > 10000) { + synchronized (buffer) { + if (buffer.size() > 10000) { + for (PreExtractRecord r : buffer) { + try { + cache.put(r.lemma, r.label, r.features); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + buffer.clear(); + } + } + } + + counter.incrementAndGet(); + } + + @Override + protected String getStatus() { + return counter.get() + " examples processed"; + } + + @Override + protected List> process( + Pair input) { + List> l = new ArrayList<>(); + l.add(input); + return l; + } + + public void finalize() throws Exception { + for (PreExtractRecord r : buffer) { + cache.put(r.lemma, r.label, r.features); + + } + log.info("Saving pruned feature cache done!"); + cache.close(); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/TextPreProcessor.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/TextPreProcessor.java new file mode 100644 index 000000000..384287ed1 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/experiment/TextPreProcessor.java @@ -0,0 +1,109 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.experiment; + +import edu.illinois.cs.cogcomp.annotation.AnnotatorException; +import edu.illinois.cs.cogcomp.annotation.AnnotatorService; +import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager; +import edu.illinois.cs.cogcomp.curator.CuratorFactory; +import edu.illinois.cs.cogcomp.edison.annotators.ClauseViewGenerator; +import edu.illinois.cs.cogcomp.edison.annotators.HeadFinderDependencyViewGenerator; +import edu.illinois.cs.cogcomp.nlp.pipeline.IllinoisPipelineFactory; +import edu.illinois.cs.cogcomp.srl.SRLProperties; +import edu.illinois.cs.cogcomp.srl.config.SrlConfigurator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TextPreProcessor { + private final static Logger log = LoggerFactory.getLogger(TextPreProcessor.class); + private static final String NAME = TextPreProcessor.class.getCanonicalName(); + + private static TextPreProcessor instance; + private final AnnotatorService annotator; + public final static String[] requiredViews = {ViewNames.POS, ViewNames.NER_CONLL, + ViewNames.LEMMA, ViewNames.SHALLOW_PARSE, ViewNames.PARSE_STANFORD}; + + /** + * requires SRLProperties to have been instantiated already + */ + public TextPreProcessor(SRLProperties config) throws Exception { + String defaultParser = config.getDefaultParser(); + boolean useCurator = config.useCurator(); + + if (useCurator) { + log.info("Using curator"); + annotator = CuratorFactory.buildCuratorClient(); + } else { + log.info("Using pipeline"); + if (!defaultParser.equals("Stanford")) { + log.error("Illinois Pipeline works only with the Stanford parser.\n" + + "Please change the 'DefaultParser' parameter in the configuration file."); + System.exit(-1); + } + annotator = + IllinoisPipelineFactory.buildPipeline(new ResourceManager(config + .getPipelineConfig())); + } + } + + public static void initialize() { + SRLProperties props = SRLProperties.getInstance(); + initialize(props); + } + + public static void initialize(ResourceManager rm) { + SRLProperties props = SRLProperties.getInstance(rm); + + initialize(props); + } + + public static void initialize(SRLProperties props) { + log.info("## initializing {}.", NAME); + try { + instance = new TextPreProcessor(props); + } catch (Exception e) { + log.error("Unable to initialize the text pre-processor"); + e.printStackTrace(); + System.exit(-1); + } + } + + public static TextPreProcessor getInstance() { + if (instance == null) { + // Start a new TextPreProcessor with default values (no Curator, no + // tokenization) and default config + initialize(new SrlConfigurator().getDefaultConfig()); + } + return instance; + } + + public TextAnnotation preProcessText(String text) throws Exception { + TextAnnotation ta; + ta = annotator.createBasicTextAnnotation("", "", text); + addViews(ta); + return ta; + } + + public void preProcessText(TextAnnotation ta) throws Exception { + addViews(ta); + } + + private void addViews(TextAnnotation ta) throws AnnotatorException { + for (String view : requiredViews) { + if (!ta.hasView(view)) + annotator.addView(ta, view); + } + if (!ta.hasView(ViewNames.CLAUSES_STANFORD)) + ta.addView(ClauseViewGenerator.STANFORD); + if (!ta.hasView(ViewNames.DEPENDENCY + ":" + ViewNames.PARSE_STANFORD)) + ta.addView(new HeadFinderDependencyViewGenerator(ViewNames.PARSE_STANFORD)); + + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/features/FeatureGenerators.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/features/FeatureGenerators.java new file mode 100644 index 000000000..8d495789d --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/features/FeatureGenerators.java @@ -0,0 +1,274 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.features; + +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.*; +import edu.illinois.cs.cogcomp.edison.features.DiscreteFeature; +import edu.illinois.cs.cogcomp.edison.features.Feature; +import edu.illinois.cs.cogcomp.edison.features.FeatureExtractor; +import edu.illinois.cs.cogcomp.edison.features.FeatureInputTransformer; +import edu.illinois.cs.cogcomp.edison.features.helpers.WordHelpers; +import edu.illinois.cs.cogcomp.edison.utilities.EdisonException; +import edu.illinois.cs.cogcomp.nlp.utilities.CollinsHeadFinder; +import edu.illinois.cs.cogcomp.nlp.utilities.POSUtils; +import edu.illinois.cs.cogcomp.nlp.utilities.ParseTreeProperties; + +import java.util.*; + +@SuppressWarnings("serial") +public class FeatureGenerators { + + public static FeatureExtractor hasVerb = new FeatureExtractor() { + + @Override + public String getName() { + return "#has-verb"; + } + + @Override + public Set getFeatures(Constituent c) throws EdisonException { + boolean hasVerb = false; + TextAnnotation ta = c.getTextAnnotation(); + + for (int i = c.getStartSpan(); i < c.getEndSpan(); i++) { + + if (POSUtils.isPOSVerb(WordHelpers.getPOS(ta, i))) { + hasVerb = true; + break; + } + } + Set feats = new HashSet<>(); + + if (hasVerb) { + feats.add(DiscreteFeature.create(getName())); + } + return feats; + } + }; + + public static final FeatureInputTransformer candidateBoundaryTransformer = + new FeatureInputTransformer() { + + @Override + public List transform(Constituent input) { + TextAnnotation ta = input.getTextAnnotation(); + + Constituent ce = + new Constituent("", "", ta, input.getEndSpan() - 1, input.getEndSpan()); + + Constituent cs = + new Constituent("", "", ta, input.getStartSpan(), + input.getStartSpan() + 1); + + new Relation("", cs, ce, 0); + + return Collections.singletonList(ce); + + } + + @Override + public String name() { + return "#b:"; + } + }; + + public static FeatureExtractor hyphenTagFeature = new FeatureExtractor() { + + @Override + public String getName() { + return "#nom-hyp"; + } + + @Override + public Set getFeatures(Constituent c) throws EdisonException { + + Set features = new HashSet<>(); + String surfaceString = c.getSurfaceForm(); + + if (surfaceString.contains("-") && c.length() == 1) { + Constituent predicate = c.getIncomingRelations().get(0).getSource(); + + String lemma = predicate.getAttribute(PredicateArgumentView.LemmaIdentifier); + + assert lemma != null; + + if (predicate.getSpan().equals(c.getSpan())) { + features.add(DiscreteFeature.create("pred-token")); + } + + String[] parts = surfaceString.split("-"); + + for (int i = 0; i < parts.length; i++) { + String part = parts[i]; + + if (part.contains(lemma)) { + features.add(DiscreteFeature.create(i + ":pred")); + } else { + String lowerCase = part.toLowerCase(); + features.add(DiscreteFeature.create(lowerCase)); + features.add(DiscreteFeature.create(i + ":" + lowerCase)); + } + } + } + + return features; + } + }; + + public static FeatureExtractor ppFeatures(final String parseViewName) { + return new FeatureExtractor() { + + @Override + public String getName() { + return "#pp-feats"; + } + + @Override + public Set getFeatures(Constituent c) throws EdisonException { + TextAnnotation ta = c.getTextAnnotation(); + + TreeView parse = (TreeView) ta.getView(parseViewName); + + Set feats = new HashSet<>(); + try { + Constituent phrase = parse.getParsePhrase(c); + // if the phrase is a PP, then the head word of its + // rightmost NP child. + + List rels = phrase.getOutgoingRelations(); + for (int i = rels.size() - 1; i >= 0; i--) { + Relation relation = rels.get(i); + if (relation == null) + continue; + Constituent target = relation.getTarget(); + if (ParseTreeProperties.isNominal(target.getLabel())) { + int head = CollinsHeadFinder.getInstance().getHeadWordPosition(phrase); + + feats.add(DiscreteFeature.create("np-head:" + + ta.getToken(head).toLowerCase())); + feats.add(DiscreteFeature.create("np-head-pos:" + + WordHelpers.getPOS(ta, head))); + + break; + } + } + + // if the phrase's parent is a PP, then the head of that PP. + Constituent parent = phrase.getIncomingRelations().get(0).getSource(); + + if (parent.getLabel().equals("PP")) { + int head = CollinsHeadFinder.getInstance().getHeadWordPosition(phrase); + feats.add(DiscreteFeature.create("p-head:" + + ta.getToken(head).toLowerCase())); + } + + } catch (EdisonException e) { + throw new RuntimeException(e); + } catch (Exception e) { + e.printStackTrace(); + } + + return feats; + } + }; + } + + public static FeatureInputTransformer getParseLeftSibling(final String parseViewName) { + return new FeatureInputTransformer() { + + @Override + public List transform(Constituent input) { + TextAnnotation ta = input.getTextAnnotation(); + + TreeView parse = (TreeView) ta.getView(parseViewName); + + List siblings = new ArrayList<>(); + try { + Constituent phrase = parse.getParsePhrase(input); + List in = phrase.getIncomingRelations(); + + if (in.size() > 0) { + Constituent prev = null; + Relation relation = in.get(0); + List outgoingRelations = + relation.getSource().getOutgoingRelations(); + + for (Relation r : outgoingRelations) { + if (r.getTarget() == phrase) { + break; + } + prev = r.getTarget(); + } + + if (prev != null) + siblings.add(prev); + } + + } catch (EdisonException e) { + throw new RuntimeException(e); + } catch (Exception e) { + e.printStackTrace(); + } + + return siblings; + } + + @Override + public String name() { + return "#lsis"; + } + }; + } + + public static FeatureInputTransformer getParseRightSibling(final String parseViewName) { + return new FeatureInputTransformer() { + + @Override + public List transform(Constituent input) { + TextAnnotation ta = input.getTextAnnotation(); + + TreeView parse = (TreeView) ta.getView(parseViewName); + + List siblings = new ArrayList<>(); + try { + Constituent phrase = parse.getParsePhrase(input); + List in = phrase.getIncomingRelations(); + + if (in.size() > 0) { + List outgoingRelations = + in.get(0).getSource().getOutgoingRelations(); + int id = -1; + for (int i = 0; i < outgoingRelations.size(); i++) { + Relation r = outgoingRelations.get(i); + if (r.getTarget() == phrase) { + id = i; + break; + } + } + + if (id >= 0 && id + 1 < outgoingRelations.size()) + siblings.add(outgoingRelations.get(id + 1).getTarget()); + } + + } catch (EdisonException e) { + throw new RuntimeException(e); + } catch (Exception e) { + e.printStackTrace(); + } + + return siblings; + } + + @Override + public String name() { + return "#rsis"; + } + }; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/features/ProjectedPath.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/features/ProjectedPath.java new file mode 100644 index 000000000..1707a8fbb --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/features/ProjectedPath.java @@ -0,0 +1,95 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.features; + +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Relation; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TreeView; +import edu.illinois.cs.cogcomp.edison.features.DiscreteFeature; +import edu.illinois.cs.cogcomp.edison.features.Feature; +import edu.illinois.cs.cogcomp.edison.features.FeatureExtractor; +import edu.illinois.cs.cogcomp.edison.features.helpers.PathFeatureHelper; +import edu.illinois.cs.cogcomp.edison.utilities.EdisonException; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class ProjectedPath implements FeatureExtractor { + + private String parseViewName; + + public ProjectedPath(String parseViewName) { + this.parseViewName = parseViewName; + } + + @Override + public Set getFeatures(Constituent c) throws EdisonException { + TextAnnotation ta = c.getTextAnnotation(); + + TreeView parse = (TreeView) ta.getView(parseViewName); + + Set feats = new HashSet<>(); + + // Clone this to avoid concurrency problems + Constituent c2 = null; + try { + c2 = parse.getParsePhrase(c).cloneForNewView(""); + } catch (Exception e) { + e.printStackTrace(); + } + + assert c2 != null; + if (!c2.getLabel().equals("VP")) + return feats; + + boolean found = false; + boolean done = false; + + while (!done) { + List rels = c2.getIncomingRelations(); + if (rels.size() == 0) + done = true; + else { + Constituent parent = rels.get(0).getSource(); + if (parent.getLabel().equals("VP")) { + found = true; + c2 = parent; + } else { + done = true; + } + } + } + + if (found) { + // Clone this to avoid concurrency problems + Constituent c1 = null; + try { + c1 = + parse.getParsePhrase(c.getIncomingRelations().get(0).getSource()) + .cloneForNewView(""); + } catch (Exception e) { + e.printStackTrace(); + } + + assert c1 != null; + String path = PathFeatureHelper.getFullParsePathString(c1, c2, 400); + feats.add(DiscreteFeature.create(path)); + } + + return feats; + + } + + @Override + public String getName() { + return "#proj-path"; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/ILPOutput.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/ILPOutput.java new file mode 100644 index 000000000..3a6f2e6e4 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/ILPOutput.java @@ -0,0 +1,135 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference; + +import edu.illinois.cs.cogcomp.infer.ilp.ILPOutputGenerator; +import edu.illinois.cs.cogcomp.infer.ilp.ILPSolver; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.learn.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class ILPOutput implements ILPOutputGenerator { + private final static Logger log = LoggerFactory.getLogger(ILPOutput.class); + private SRLManager manager; + + public ILPOutput(SRLManager manager) { + this.manager = manager; + + } + + public SRLSentenceStructure getOutput(ILPSolver xmp, + InferenceVariableLexManager variableManager, IInstance ins) { + + SRLSentenceInstance instance = (SRLSentenceInstance) ins; + + int numPredicates = instance.numPredicates(); + List output = new ArrayList<>(); + + for (int predicateId = 0; predicateId < numPredicates; predicateId++) { + + SRLPredicateInstance x = instance.predicates.get(predicateId); + + SRLMulticlassInstance senseInstance = x.getSenseInstance(); + + String lemma = senseInstance.getPredicateLemma(); + + int senseLabel = + getSenseLabelFromPrediction(xmp, variableManager, manager, predicateId, lemma); + + int[] argLabels = + getArgumentLabels(xmp, variableManager, manager, predicateId, x, lemma); + + output.add(new SRLPredicateStructure(x, argLabels, senseLabel, manager)); + } + + return new SRLSentenceStructure(instance, output); + } + + private int[] getArgumentLabels(ILPSolver xmp, InferenceVariableLexManager variableManager, + SRLManager manager, int predicateId, SRLPredicateInstance x, String lemma) { + Set legalArgsSet = manager.getLegalArguments(lemma); + + List candidateInstances = x.getCandidateInstances(); + + int numCandidates = candidateInstances.size(); + + int[] argLabels = new int[numCandidates]; + + log.debug("Getting output for " + x); + + for (int candidateId = 0; candidateId < numCandidates; candidateId++) { + + log.debug("Considering {}", x.getCandidateInstances().get(candidateId)); + String label = null; + for (String l : legalArgsSet) { + String variableIdentifier = + SRLILPInference.getArgumentVariableIdentifier( + manager.getPredictedViewName(), predicateId, candidateId, l); + int var = variableManager.getVariable(variableIdentifier); + + if (var < 0) + continue; + + boolean value = xmp.getBooleanValue(var); + + log.debug(" {}: {}", l, value); + + if (value) { + + label = l; + + log.debug("Variable: " + var + ", label = " + label + ", label-id = " + + manager.getArgumentId(label)); + break; + } + } + + assert label != null; + argLabels[candidateId] = manager.getArgumentId(label); + + log.debug("Prediction for {}: {}", x.getCandidateInstances().get(candidateId), label); + + } + return argLabels; + } + + private int getSenseLabelFromPrediction(ILPSolver xmp, + InferenceVariableLexManager variableManager, SRLManager manager, int predicateId, + String lemma) { + String sense = null; + Set validSenseLabels = manager.getLegalSenses(lemma); + for (String label : validSenseLabels) { + String varName = + SRLILPInference.getSenseVariableIdentifier(manager.getPredictedViewName(), + predicateId, label); + + int var = variableManager.getVariable(varName); + + if (var < 0) + continue; + + if (xmp.getBooleanValue(var)) { + sense = label; + break; + } + } + + assert sense != null; + + int senseLabel = manager.getSenseId(sense); + return senseLabel; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLConstraints.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLConstraints.java new file mode 100644 index 000000000..faf702cfb --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLConstraints.java @@ -0,0 +1,77 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference; + +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.inference.constraints.*; + +public enum SRLConstraints { + + noOverlappingArguments { + @Override + public SRLILPConstraintGenerator getGenerator(SRLManager manager) { + return new NoOverlapConstraint(manager); + } + }, + noDuplicateCore { + @Override + public SRLILPConstraintGenerator getGenerator(SRLManager manager) { + return new NoDuplicateCoreConstraint(manager); + } + }, + CArgumentConstraint { + @Override + public SRLILPConstraintGenerator getGenerator(SRLManager manager) { + return new CArgConstraints(manager); + } + }, + RArgumentConstraint { + @Override + public SRLILPConstraintGenerator getGenerator(SRLManager manager) { + return new RArgConstraints(manager); + } + }, + predicateSense { + @Override + public SRLILPConstraintGenerator getGenerator(SRLManager manager) { + return new PredicateSenseConstraints(manager); + } + }, + beVerbConstraint { + @Override + public SRLILPConstraintGenerator getGenerator(SRLManager manager) { + return new BeVerbConstraints(manager); + } + }, + supportVerbConstraint { + @Override + public SRLILPConstraintGenerator getGenerator(SRLManager manager) { + return new SupportVerbConstraint(manager); + } + }, + noCrossArgumentExclusiveOverlap { + @Override + public SRLILPConstraintGenerator getGenerator(SRLManager manager) { + return new CrossArgumentExclusiveOverlap(manager); + } + }, + crossArgumentRetainedModifiers { + @Override + public SRLILPConstraintGenerator getGenerator(SRLManager manager) { + return new CrossArgumentRetainedModifiers(manager); + } + }, + atLeastOneCoreArgument { + @Override + public SRLILPConstraintGenerator getGenerator(SRLManager manager) { + return new AtLeastOneCoreArgument(manager); + } + }; + + public abstract SRLILPConstraintGenerator getGenerator(SRLManager manager); +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLILPConstraintGenerator.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLILPConstraintGenerator.java new file mode 100644 index 000000000..7fc4bb3fe --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLILPConstraintGenerator.java @@ -0,0 +1,59 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference; + +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraintGenerator; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; + +import java.util.ArrayList; +import java.util.List; + +abstract public class SRLILPConstraintGenerator extends ILPConstraintGenerator { + + protected SRLManager manager; + + public SRLILPConstraintGenerator(SRLManager manager, String name, boolean delayedConstraint) { + super(name, delayedConstraint); + this.manager = manager; + } + + protected Pair cleanupVariables(int[] vars, double[] coefs) { + List c = new ArrayList<>(); + List v = new ArrayList<>(); + + for (int i = 0; i < vars.length; i++) { + if (vars[i] >= 0) { + v.add(vars[i]); + c.add(coefs[i]); + + } + } + + int[] v1 = new int[v.size()]; + double[] d = new double[c.size()]; + + for (int i = 0; i < c.size(); i++) { + v1[i] = v.get(i); + d[i] = c.get(i); + } + + return new Pair<>(v1, d); + + } + + protected int getArgumentVariable(InferenceVariableLexManager variables, String type, + int predicateId, int candidateId, String arg) { + + String identifier = + SRLILPInference.getArgumentVariableIdentifier(type, predicateId, candidateId, arg); + + return variables.getVariable(identifier); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLILPInference.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLILPInference.java new file mode 100644 index 000000000..7c62f1e6f --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLILPInference.java @@ -0,0 +1,243 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference; + +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.infer.ilp.*; +import edu.illinois.cs.cogcomp.infer.ilp.ILPSolverFactory.SolverType; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceStructure; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +final public class SRLILPInference extends AbstractILPInference { + + final static boolean DEBUG = false; + + private final static Logger log = LoggerFactory.getLogger(SRLILPInference.class); + + public final SRLSentenceInstance instance; + + protected TextAnnotation ta; + + protected final SRLManager manager; + + private String viewName; + + private final int numPredicates; + + public boolean debugMode = false; + + private ILPOutput outputGenerator; + + public SRLILPInference(ILPSolverFactory solverFactory, SRLManager manager, + List predicates) throws Exception { + super(solverFactory, DEBUG); + this.manager = manager; + + this.outputGenerator = new ILPOutput(manager); + + List instances = new ArrayList<>(); + for (Constituent predicate : predicates) { + Constituent predicateClone = predicate.cloneForNewView(predicate.getViewName()); + SRLPredicateInstance x; + + assert predicateClone.hasAttribute(PredicateArgumentView.LemmaIdentifier); + + x = new SRLPredicateInstance(predicateClone, manager, manager.getArgumentIdentifier()); + x.cacheAllFeatureVectors(false); + + instances.add(x); + } + + ta = predicates.get(0).getTextAnnotation(); + + viewName = manager.getPredictedViewName(); + + instance = new SRLSentenceInstance(instances); + numPredicates = instance.numPredicates(); + } + + @Override + protected void initializeSolver(ILPSolver xmp, InferenceVariableLexManager variableManager) { + if (this.solverFactory.type == SolverType.JLISCuttingPlaneGurobi) { + JLISCuttingPlaneILPSolverGurobi s = (JLISCuttingPlaneILPSolverGurobi) xmp; + s.setInput(instance); + s.setVariableManager(variableManager); + s.setOutputGenerator(outputGenerator); + } + } + + @Override + protected SRLSentenceStructure getOutput(ILPSolver xmp, + InferenceVariableLexManager variableManager) throws Exception { + return outputGenerator.getOutput(xmp, variableManager, this.instance); + } + + protected void addConstraints(ILPSolver xmp, List constraints, + String debugMessage) { + log.debug(debugMessage); + + for (ILPConstraint c : constraints) { + addConstraint(xmp, c); + } + } + + @Override + protected void addConstraints(ILPSolver xmp, InferenceVariableLexManager variableManager) { + Set constraints = manager.getConstraints(); + if (constraints.size() == 0) + log.error("No constraints found. This can't be right."); + + for (SRLConstraints cc : constraints) { + + if (debugMode) { + System.out.println("Adding constriant: " + cc); + } + + SRLILPConstraintGenerator c = cc.getGenerator(manager); + + if (c.isDelayedConstraint() && xmp instanceof JLISCuttingPlaneILPSolverGurobi) { + ((JLISCuttingPlaneILPSolverGurobi) xmp).addCuttingPlaneConstraintGenerator(c); + } else { + List cs = c.getILPConstraints(instance, variableManager); + addConstraints(xmp, cs, c.name); + } + } + } + + @Override + protected void addVariables(ILPSolver xmp, InferenceVariableLexManager variableManager) { + assert xmp != null; + + for (int predicateId = 0; predicateId < numPredicates; predicateId++) { + SRLPredicateInstance x = instance.predicates.get(predicateId); + + SRLMulticlassInstance senseX = x.getSenseInstance(); + List candidates = x.getCandidateInstances(); + + String lemma = senseX.getPredicateLemma(); + assert lemma != null; + + log.debug("Adding variables for " + lemma); + + Set legalArgs = manager.getLegalArguments(lemma); + + Set set; + for (int candidateId = 0; candidateId < candidates.size(); candidateId++) { + + SRLMulticlassInstance cX = candidates.get(candidateId); + double[] scores = manager.getScores(cX, Models.Classifier, true); + + log.debug("\tCandidate = " + cX.toString()); + + assert scores.length == manager.getAllArguments().size(); + + double idScore; + double[] sc = manager.getScores(cX, Models.Identifier, true); + idScore = sc[1] - sc[0]; + + + set = new HashSet<>(); + for (int labelId = 0; labelId < scores.length; labelId++) { + String label = manager.getArgument(labelId); + + assert label != null : labelId + " is a null object!"; + + if (!legalArgs.contains(label)) + continue; + + double score = scores[labelId]; + if (label.equals(SRLManager.NULL_LABEL)) + score -= idScore; + + String variableIdentifier = + getArgumentVariableIdentifier(viewName, predicateId, candidateId, label); + + int var = xmp.addBooleanVariable(score); + variableManager.addVariable(variableIdentifier, var); + + set.add(var); + + log.debug("Arg variable: " + score + " " + variableIdentifier + " " + var + " " + + label); + } + + log.debug("Adding unique arg label constraint for {}", candidateId); + addUniqueLabelConstraint(xmp, set); + + } + + double[] senseScores = manager.getScores(senseX, Models.Sense, true); + + set = new HashSet<>(); + + for (int senseId = 0; senseId < senseScores.length; senseId++) { + + if (!manager.isValidSense(lemma, senseId)) + continue; + + String label = manager.getSense(senseId); + + double score = senseScores[senseId]; + + String variableIdentifier = + getSenseVariableIdentifier(viewName, predicateId, label); + + int var = xmp.addBooleanVariable(score); + variableManager.addVariable(variableIdentifier, var); + + set.add(var); + log.debug("Sense variable: " + score + " " + variableIdentifier + " " + var + " " + + label); + } + + log.debug("Adding unique sense label constraint"); + addUniqueLabelConstraint(xmp, set); + + } + + assert variableManager.size() > 0 : "No varaibles added for " + this.ta; + } + + private void addUniqueLabelConstraint(ILPSolver xmp, Set set) { + int[] vars = new int[set.size()]; + double[] coeffs = new double[set.size()]; + int i = 0; + for (int v : set) { + vars[i] = v; + coeffs[i++] = 1.0; + } + + addEqualityConstraint(xmp, vars, coeffs, 1.0); + } + + public static String getArgumentVariableIdentifier(String type, int predicateId, + int candidateId, String label) { + return type + ":" + predicateId + ":" + candidateId + ":" + label; + } + + public static String getSenseVariableIdentifier(String type, int predicateId, String label) { + return type + ":sense:" + predicateId + ":" + label; + } + + public PredicateArgumentView getOutputView() throws Exception { + return runInference().getView(manager, ta); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLMulticlassInference.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLMulticlassInference.java new file mode 100644 index 000000000..3b809add1 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/SRLMulticlassInference.java @@ -0,0 +1,127 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference; + +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.sl.core.AbstractInferenceSolver; +import edu.illinois.cs.cogcomp.sl.util.WeightVector; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassLabel; + +@SuppressWarnings("serial") +public class SRLMulticlassInference extends AbstractInferenceSolver { + + private final Models type; + private final SRLManager manager; + private boolean stepThrough; + + public SRLMulticlassInference(SRLManager manager, Models type) { + this.manager = manager; + this.type = type; + } + + @Override + public IStructure getLossAugmentedBestStructure(WeightVector weight, IInstance ins, + IStructure goldStructure) throws Exception { + SRLMulticlassInstance x = (SRLMulticlassInstance) ins; + SRLMulticlassLabel yGold = null; + if (goldStructure != null) + yGold = (SRLMulticlassLabel) goldStructure; + + int numLabels = manager.getNumLabels(type); + assert numLabels > 0; + + if (type == Models.Identifier) + assert numLabels == 2; + + double max = Double.NEGATIVE_INFINITY; + SRLMulticlassLabel best = null; + double loss = 0; + + if (stepThrough) { + System.out.println("Stepping through inference"); + } + + for (int label = 0; label < numLabels; label++) { + + if (stepThrough) { + System.out.println("Label: " + manager.getArgument(label)); + } + + if (!manager.isValidLabel(x, type, label)) { + if (stepThrough) + System.out.println("Label is not valid for " + x.getPredicateLemma()); + continue; + } + + SRLMulticlassLabel y = new SRLMulticlassLabel(label, type, manager); + + double score = + weight.dotProduct(x.getCachedFeatureVector(type), + label * manager.getModelInfo(type).getLexicon().size()); + + if (stepThrough) + System.out.println("\t Score = " + score); + + double l = 0; + if (goldStructure != null) { + if (yGold.getLabel() != label) + l++; + } + + if (score + l > max + loss) { + max = score; + loss = l; + best = y; + if (stepThrough) + System.out.println("\t\tBest so far"); + } + + } + + if (best == null) { + if (type == Models.Sense) { + System.out.println(ins); + System.out.println(manager.getLegalSenses(x.getPredicateLemma())); + } + + } + + if (stepThrough) { + System.out.println("\nBest label: " + manager.getArgument(best.getLabel())); + } + + assert best != null : type + "\t" + ins; + // return new Pair(best, loss); + return best; + } + + @Override + public float getLoss(IInstance ins, IStructure gold, IStructure pred) { + SRLMulticlassLabel yGold = (SRLMulticlassLabel) gold; + SRLMulticlassLabel ypred = (SRLMulticlassLabel) pred; + float l = 0; + if (yGold.getLabel() != ypred.getLabel()) + l++; + + return l; + } + + @Override + public IStructure getBestStructure(WeightVector weight, IInstance ins) throws Exception { + return getLossAugmentedBestStructure(weight, ins, null); + } + + @Override + public Object clone() { + return new SRLMulticlassInference(manager, type); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/AtLeastOneCoreArgument.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/AtLeastOneCoreArgument.java new file mode 100644 index 000000000..ccaa577db --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/AtLeastOneCoreArgument.java @@ -0,0 +1,117 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference.constraints; + +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraint; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.inference.SRLConstraints; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPConstraintGenerator; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateStructure; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceStructure; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * Every verb should have at leaast one argument that is one of A0, A1 or A2. Holds only for verb. + * + * @author Vivek Srikumar + * + */ +public class AtLeastOneCoreArgument extends SRLILPConstraintGenerator { + + public AtLeastOneCoreArgument(SRLManager manager) { + super(manager, SRLConstraints.atLeastOneCoreArgument.name(), true); + } + + @Override + public List getILPConstraints(IInstance x, InferenceVariableLexManager variables) { + return getViolatedILPConstraints(x, null, variables); + } + + @Override + public List getViolatedILPConstraints(IInstance ins, IStructure s, + InferenceVariableLexManager variables) { + SRLSentenceInstance x = (SRLSentenceInstance) ins; + SRLSentenceStructure y = (SRLSentenceStructure) s; + + List list = new ArrayList<>(); + + for (int predicateId = 0; predicateId < x.numPredicates(); predicateId++) { + SRLPredicateInstance xp = x.predicates.get(predicateId); + + SRLPredicateStructure yp = y == null ? null : y.ys.get(predicateId); + + list.addAll(addPredicateConstraints(manager, variables, manager.getCoreArguments(), + predicateId, xp, yp)); + } + + return list; + } + + private List addPredicateConstraints(SRLManager manager, + InferenceVariableLexManager variables, Set coreArgs, int predicateId, + SRLPredicateInstance x, SRLPredicateStructure y) { + + String type = manager.getPredictedViewName(); + int numCandidates = x.getCandidateInstances().size(); + + List list = new ArrayList<>(); + + int[] vars = new int[numCandidates * coreArgs.size()]; + double[] coefs = new double[vars.length]; + + boolean found = false; + + int id = 0; + + for (String coreArgument : coreArgs) { + + int argId = manager.getArgumentId(coreArgument); + + for (int candidateId = 0; candidateId < numCandidates; candidateId++) { + + vars[id] = + getArgumentVariable(variables, type, predicateId, candidateId, coreArgument); + + coefs[id] = 1; + + id++; + + if (y != null) { + if (y.getArgLabel(candidateId) == argId) { + found = true; + break; + } + } + } + + if (found) + break; + } + + if (!found) { + Pair cleanedVar = cleanupVariables(vars, coefs); + vars = cleanedVar.getFirst(); + coefs = cleanedVar.getSecond(); + + if (vars.length > 0) + list.add(new ILPConstraint(vars, coefs, 1.0, ILPConstraint.GREATER_THAN)); + } + + return list; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/BeVerbConstraints.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/BeVerbConstraints.java new file mode 100644 index 000000000..0146b59ca --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/BeVerbConstraints.java @@ -0,0 +1,122 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference.constraints; + +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.edison.features.helpers.WordHelpers; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraint; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPConstraintGenerator; +import edu.illinois.cs.cogcomp.srl.learn.*; + +import java.util.*; + +public class BeVerbConstraints extends SRLILPConstraintGenerator { + + public static final String name = "beVerbConstraints"; + + private final static Set validArgs = new HashSet<>(Arrays.asList("A1", "AA", "AM", + "AM-ADV", "AM-CAU", "AM-DIR", "AM-DIS", "AM-EXT", "AM-LOC", "AM-MNR", "AM-MOD", + "AM-NEG", "AM-PNC", "AM-PRD", "AM-REC", "AM-TMP", "C-A0", "C-A1", "C-A2", "C-A3", + "C-AM-ADV", "C-AM-CAU", "C-AM-DIS", "C-AM-EXT", "C-AM-LOC", "C-AM-MNR", "R-A0", "R-A1", + "R-A2", "R-A3", "R-AA", "R-AM-ADV", "R-AM-LOC", "R-AM-MNR", "R-AM-PNC", "R-AM-TMP", + "V", "C-V", SRLManager.NULL_LABEL, "C-A4", "C-A5", "C-AM-DIR", "C-AM-NEG", "C-AM-PNC", + "C-AM-TMP", "R-A4", "R-AA", "R-AM-CAU", "R-AM-DIR", "R-AM-EXT")); + + public BeVerbConstraints(SRLManager manager) { + super(manager, name, false); + } + + @Override + public List getILPConstraints(IInstance x, InferenceVariableLexManager variables) { + return getViolatedILPConstraints(x, null, variables); + } + + @Override + public List getViolatedILPConstraints(IInstance ins, IStructure s, + InferenceVariableLexManager variables) { + + SRLSentenceInstance x = (SRLSentenceInstance) ins; + SRLSentenceStructure y = (SRLSentenceStructure) s; + + List list = new ArrayList<>(); + + for (int predicateId = 0; predicateId < x.numPredicates(); predicateId++) { + + SRLPredicateInstance xp = x.predicates.get(predicateId); + + SRLMulticlassInstance senseInstance = xp.getSenseInstance(); + + String lemma = senseInstance.getPredicateLemma(); + + if (!lemma.equals("be")) + return list; + + Constituent predicate = senseInstance.getConstituent(); + + TextAnnotation ta = predicate.getTextAnnotation(); + + int predicateToken = senseInstance.getConstituent().getStartSpan(); + + Set allowedArgs = new HashSet<>(validArgs); + + boolean done = false; + if (predicateToken + 1 < ta.getSentence(predicate.getSentenceId()).getEndSpan()) { + String nextToken = WordHelpers.getLemma(ta, predicateToken + 1); + + if (nextToken.toLowerCase().equals("like")) { + allowedArgs.add("A0"); + + } + done = true; + } + + if (!done) + allowedArgs.add("A2"); + + SRLPredicateStructure yp = (y == null ? null : y.ys.get(predicateId)); + + for (int candidateId = 0; candidateId < xp.getCandidateInstances().size(); candidateId++) { + + if (y != null) { + int argLabel = yp.getArgLabel(candidateId); + if (!allowedArgs.contains(manager.getArgument(argLabel))) { + continue; + } + } + + int[] vars = new int[allowedArgs.size()]; + double[] coef = new double[allowedArgs.size()]; + + int id = 0; + for (String label : allowedArgs) { + + vars[id] = + getArgumentVariable(variables, manager.getPredictedViewName(), + predicateId, candidateId, label); + coef[id] = 1.0; + id++; + } + + Pair pair = cleanupVariables(vars, coef); + + vars = pair.getFirst(); + coef = pair.getSecond(); + + if (vars.length > 0) + list.add(new ILPConstraint(vars, coef, 1.0, ILPConstraint.EQUAL)); + } + } + return list; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/CArgConstraints.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/CArgConstraints.java new file mode 100644 index 000000000..f21559584 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/CArgConstraints.java @@ -0,0 +1,179 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference.constraints; + +import edu.illinois.cs.cogcomp.core.datastructures.IntPair; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraint; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.core.SRLType; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPConstraintGenerator; +import edu.illinois.cs.cogcomp.srl.learn.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class CArgConstraints extends SRLILPConstraintGenerator { + private final static Logger log = LoggerFactory.getLogger(CArgConstraints.class); + + public static final String name = "CArgumentConstraint"; + + public CArgConstraints(SRLManager manager) { + super(manager, name, true); + } + + @Override + public List getILPConstraints(IInstance x, InferenceVariableLexManager variables) { + log.debug("Getting ILP constraints for {}", name); + return getViolatedILPConstraints(x, null, variables); + } + + @Override + public List getViolatedILPConstraints(IInstance ins, IStructure s, + InferenceVariableLexManager variables) { + + SRLSentenceInstance x = (SRLSentenceInstance) ins; + SRLSentenceStructure y = (SRLSentenceStructure) s; + + List list = new ArrayList<>(); + log.debug("Adding C-Arg constriants"); + + int numPredicates = x.numPredicates(); + for (int predicateId = 0; predicateId < numPredicates; predicateId++) { + SRLPredicateInstance xp = x.predicates.get(predicateId); + + log.debug("Predicate: " + x); + + SRLPredicateStructure yp = y == null ? null : y.ys.get(predicateId); + list.addAll(getPredicateCArgConstraints(manager, variables, predicateId, xp, yp)); + } + + return list; + } + + private List getPredicateCArgConstraints(SRLManager manager, + InferenceVariableLexManager variables, int predicateId, SRLPredicateInstance x, + SRLPredicateStructure y) { + + List candidateIds = getSortedCandidates(x); + + List list = new ArrayList<>(); + for (String arg : manager.getCoreArguments()) { + list.addAll(getCArgConstraint(manager, predicateId, variables, candidateIds, arg, x, y)); + } + + for (String arg : manager.getModifierArguments()) { + + list.addAll(getCArgConstraint(manager, predicateId, variables, candidateIds, arg, x, y)); + } + + if (manager.getSRLType() == SRLType.Nom) + list.addAll(getCArgConstraint(manager, predicateId, variables, candidateIds, "SUP", x, + y)); + + return list; + } + + private List getCArgConstraint(SRLManager manager, int predicateId, + InferenceVariableLexManager variables, List candidateIds, String arg, + SRLPredicateInstance x, SRLPredicateStructure y) { + List list = new ArrayList<>(); + + String cArg = "C-" + arg; + int numCandidates = x.getCandidateInstances().size(); + String type = manager.getPredictedViewName(); + + for (int position = 0; position < numCandidates - 1; position++) { + + int[] vars = new int[position + 2]; + double[] coefs = new double[position + 2]; + + int cCandidateId = candidateIds.get(position + 1); + vars[position + 1] = + getArgumentVariable(variables, type, predicateId, cCandidateId, cArg); + + if (vars[position + 1] < 0) + continue; + + coefs[position + 1] = -1; + + if (y != null) { + if (y.getArgLabel(cCandidateId) != manager.getArgumentId(cArg)) + continue; + } + + boolean foundArg = false; + for (int i = 0; i < position + 1; i++) { + int candidateId = candidateIds.get(i); + + vars[i] = getArgumentVariable(variables, type, predicateId, candidateId, arg); + coefs[i] = 1; + + if (y != null) { + if (y.getArgLabel(candidateId) == manager.getArgumentId(arg)) + foundArg = true; + } + } + + if (y != null && foundArg) + continue; + + Pair cleanedVar = cleanupVariables(vars, coefs); + vars = cleanedVar.getFirst(); + coefs = cleanedVar.getSecond(); + + if (vars.length > 0) { + ILPConstraint constraint = + new ILPConstraint(vars, coefs, 0, ILPConstraint.GREATER_THAN); + + log.debug(arg + ": " + constraint); + + list.add(constraint); + + } + } + return list; + } + + private List getSortedCandidates(SRLPredicateInstance x) { + List candidateIds = new ArrayList<>(); + + final List candidates = x.getCandidateInstances(); + int numCandidates = candidates.size(); + + for (int candidateId = 0; candidateId < numCandidates; candidateId++) { + candidateIds.add(candidateId); + } + + Collections.sort(candidateIds, new Comparator() { + public int compare(Integer o1, Integer o2) { + + IntPair span1 = candidates.get(o1).getConstituent().getSpan(); + IntPair span2 = candidates.get(o2).getConstituent().getSpan(); + + int start1 = span1.getFirst(); + int start2 = span2.getFirst(); + if (start1 < start2) + return -1; + else if (start1 == start2) + return 0; + else + return 1; + } + }); + return candidateIds; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/CrossArgumentExclusiveOverlap.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/CrossArgumentExclusiveOverlap.java new file mode 100644 index 000000000..cf986062e --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/CrossArgumentExclusiveOverlap.java @@ -0,0 +1,138 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference.constraints; + +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Queries; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraint; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.inference.SRLConstraints; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPConstraintGenerator; +import edu.illinois.cs.cogcomp.srl.learn.SRLMulticlassInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateStructure; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceStructure; + +import java.util.ArrayList; +import java.util.List; + +/** + * This comes from section 7.1 of "Combination Strategies for Semantic Role Labeling": Arguments + * fror two different verbs can not overlap, but they can embed. + *

+ * This is almost true in the verb SRL training set, but not so for nom. + * + * @author svivek + * + */ +public class CrossArgumentExclusiveOverlap extends SRLILPConstraintGenerator { + + private int nullId; + + public CrossArgumentExclusiveOverlap(SRLManager manager) { + super(manager, SRLConstraints.noCrossArgumentExclusiveOverlap.name(), false); + nullId = manager.getArgumentId(SRLManager.NULL_LABEL); + + } + + @Override + public List getILPConstraints(IInstance x, InferenceVariableLexManager variables) { + return getViolatedILPConstraints(x, null, variables); + } + + @Override + public List getViolatedILPConstraints(IInstance ins, IStructure s, + InferenceVariableLexManager variables) { + + SRLSentenceInstance x = (SRLSentenceInstance) ins; + SRLSentenceStructure y = (SRLSentenceStructure) s; + + int numPredicates = x.numPredicates(); + + List constraints = new ArrayList<>(); + + for (int predicateId = 0; predicateId < numPredicates; predicateId++) { + + List candidateInstances = + x.predicates.get(predicateId).getCandidateInstances(); + int numCandidates = candidateInstances.size(); + + SRLPredicateStructure predicateStructure = null; + if (y != null) { + predicateStructure = y.ys.get(predicateId); + } + + for (int candidateId = 0; candidateId < numCandidates; candidateId++) { + + if (y != null) { + if (predicateStructure.getArgLabel(candidateId) == nullId) + continue; + } + + addCandidateConstraints(predicateId, candidateId, + candidateInstances.get(candidateId).getConstituent(), x, y, constraints, + variables); + + } + + } + + return constraints; + } + + private void addCandidateConstraints(int predicateId, int candidateId, Constituent candidate, + SRLSentenceInstance x, SRLSentenceStructure y, List constraints, + InferenceVariableLexManager variables) { + + int nullVar = + this.getArgumentVariable(variables, manager.getPredictedViewName(), predicateId, + candidateId, SRLManager.NULL_LABEL); + + for (int otherPredicateId = predicateId + 1; otherPredicateId < x.numPredicates(); otherPredicateId++) { + + List candidateInstances = + x.predicates.get(otherPredicateId).getCandidateInstances(); + int numCandidates = candidateInstances.size(); + + SRLPredicateStructure otherPredicateStructure = null; + if (y != null) { + otherPredicateStructure = y.ys.get(predicateId); + } + + for (int otherCandidateId = 0; otherCandidateId < numCandidates; otherCandidateId++) { + + Constituent otherConstituent = + candidateInstances.get(otherCandidateId).getConstituent(); + + int otherLabel = -1; + if (y != null) { + otherLabel = otherPredicateStructure.getArgLabel(candidateId); + + if (otherLabel == nullId) + continue; + } + + if (!Queries.exclusivelyOverlaps(candidate).transform(otherConstituent)) + continue; + + int otherNullVar = + this.getArgumentVariable(variables, manager.getPredictedViewName(), + otherPredicateId, otherCandidateId, SRLManager.NULL_LABEL); + + // at least one of these two variables should be null. + + constraints.add(new ILPConstraint(new int[] {nullVar, otherNullVar}, new double[] { + 1, 1}, 1, ILPConstraint.GREATER_THAN)); + } + + } + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/CrossArgumentRetainedModifiers.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/CrossArgumentRetainedModifiers.java new file mode 100644 index 000000000..49510d4c4 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/CrossArgumentRetainedModifiers.java @@ -0,0 +1,337 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference.constraints; + +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.io.IOUtils; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraint; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.core.SRLType; +import edu.illinois.cs.cogcomp.srl.data.LegalArguments; +import edu.illinois.cs.cogcomp.srl.inference.SRLConstraints; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPConstraintGenerator; + +import edu.illinois.cs.cogcomp.srl.learn.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URL; +import java.util.*; + +/** + * Once a modifier, always a modifier. A modifier for one predicate can only be a modifier for + * another. The label need not be the same, though. + *

+ * For verbs, the first argument cannot be an AM-PNC and the second argument can either have the + * same label OR the label of the second argument could be X with a sibling C-X OR the label of the + * second argument could be C-X. + * + * @author Vivek Srikumar + * + */ +public class CrossArgumentRetainedModifiers extends SRLILPConstraintGenerator { + + private final static Logger log = LoggerFactory.getLogger(CrossArgumentRetainedModifiers.class); + + private static Map> verbViolations, nomViolations; + + public CrossArgumentRetainedModifiers(SRLManager manager) { + super(manager, SRLConstraints.crossArgumentRetainedModifiers.name(), true); + + if (manager.getSRLType() == SRLType.Verb) { + if (verbViolations == null) { + synchronized (log) { + if (verbViolations == null) { + try { + verbViolations = loadValidViolations(manager.getSRLType()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + } + } else { + if (nomViolations == null) { + synchronized (log) { + if (nomViolations == null) { + try { + nomViolations = loadValidViolations(manager.getSRLType()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + } + } + } + + @Override + public List getILPConstraints(IInstance x, InferenceVariableLexManager variables) { + return getViolatedILPConstraints(x, null, variables); + } + + @Override + public List getViolatedILPConstraints(IInstance ins, IStructure s, + InferenceVariableLexManager variables) { + SRLSentenceInstance x = (SRLSentenceInstance) ins; + SRLSentenceStructure y = (SRLSentenceStructure) s; + + int numPredicates = x.numPredicates(); + + // WARNING: Hairy code ahead. Read the comments carefully. + + List constraints = new ArrayList<>(); + + for (int predicateId = 0; predicateId < numPredicates; predicateId++) { + + List candidateInstances = + x.predicates.get(predicateId).getCandidateInstances(); + int numCandidates = candidateInstances.size(); + + SRLPredicateStructure predicateStructure = null; + if (y != null) { + predicateStructure = y.ys.get(predicateId); + } + + for (int candidateId = 0; candidateId < numCandidates; candidateId++) { + + // for all modifier arguments, + for (String modifier : manager.getModifierArguments()) { + if (predicateStructure != null) { + // if this is not the modifier being verified, don't + // add any constraint + if (predicateStructure.getArgLabel(candidateId) != manager + .getArgumentId(modifier)) + continue; + } + + addModifierConstraint(predicateId, candidateId, modifier, candidateInstances + .get(candidateId).getConstituent(), x, y, constraints, variables); + + } // foreach modifier + } + } + return constraints; + } + + private void addModifierConstraint(int predicateId, int candidateId, String modifier, + Constituent candidate, SRLSentenceInstance x, SRLSentenceStructure y, + List constraints, InferenceVariableLexManager variables) { + int modVar = + this.getArgumentVariable(variables, manager.getPredictedViewName(), predicateId, + candidateId, modifier); + + // If argument is a modifier, then a candidate for another preidcate + // that has the same span can ONLY have one of a few labels. Let's + // enumerate them. + + for (int otherPredicateId = 0; otherPredicateId < x.numPredicates(); otherPredicateId++) { + + if (otherPredicateId == predicateId) + continue; + + SRLPredicateInstance otherPredicateInstance = x.predicates.get(otherPredicateId); + List candidateInstances = + otherPredicateInstance.getCandidateInstances(); + + String otherLemma = otherPredicateInstance.getSenseInstance().getPredicateLemma(); + + int numCandidates = candidateInstances.size(); + + SRLPredicateStructure otherPredicateStructure = null; + if (y != null) { + otherPredicateStructure = y.ys.get(otherPredicateId); + } + + for (int otherCandidateId = 0; otherCandidateId < numCandidates; otherCandidateId++) { + // This constraint only applies for candidates that have teh + // same span. + Constituent otherConstituent = + candidateInstances.get(otherCandidateId).getConstituent(); + if (!candidate.getSpan().equals(otherConstituent.getSpan())) + continue; + + int label = -1; + if (y != null) { + assert otherPredicateStructure != null; + label = otherPredicateStructure.getArgLabel(otherCandidateId); + } + + Set vars = new HashSet<>(); + + // go over all the options. At each point, check if the + // constraint is satisfied. If so, don't add the constraint. + + // 1. It can be the modifier + if (checkAddConstraint(variables, otherPredicateId, otherCandidateId, label, vars, + modifier)) + continue; + + // 2. It can be null; + if (checkAddConstraint(variables, otherPredicateId, otherCandidateId, label, vars, + SRLManager.NULL_LABEL)) + continue; + + // 3. If the "otherlemma" is one of the allowed violations, it + // can be the label specified in the data. The allowed + // violations are generated by the function + // generateLegalArgumentsFile below. + if (checkAddValidViolationConstraint(modifier, variables, otherPredicateId, + otherCandidateId, otherLemma, label, vars)) + continue; + + // 4. If the other label is one of the C-args + boolean ignore = false; + for (String arg : manager.getAllArguments()) { + if (checkAddConstraint(variables, otherPredicateId, otherCandidateId, label, + vars, "C-" + arg)) { + ignore = true; + break; + } + } + if (ignore) + continue; + + if (manager.getSRLType() == SRLType.Nom) { + // 5. For nom, the only "other modifier label" allowed: If + // this label is AM-MNR, other can be AM-ADV. and vice + // versa. Because this is not a very clear distinction in + // the training set. + if (modifier.equals("AM-MNR")) { + if (checkAddConstraint(variables, otherPredicateId, otherCandidateId, + label, vars, "AM-ADV")) + continue; + } + + if (modifier.equals("AM-ADV")) { + if (checkAddConstraint(variables, otherPredicateId, otherCandidateId, + label, vars, "AM-MNR")) + continue; + } + } + + /* + * XXX: Not doing this for now. + * + * // 6. The most complicated one. The other argument can be any // argument label, + * provided some other candidate for that // predicate is a C-arg for that label. + * Checking for this // involves looping over all argument labels, and other // + * candidates and finally adding the constraint. + * + * checkCArgConstraints(variables, modVar, vars, otherPredicateId, otherCandidateId, + * otherPredicateStructure, label, constraints); + */ + + int[] v = new int[vars.size() + 1]; + double[] c = new double[vars.size() + 1]; + v[0] = modVar; + c[0] = -1; + + int count = 1; + for (int i : vars) { + v[count] = i; + c[count] = 1; + count++; + } + + Pair clean = cleanupVariables(v, c); + + if (clean.getFirst().length > 0) + constraints.add(new ILPConstraint(clean.getFirst(), clean.getSecond(), 0, + ILPConstraint.GREATER_THAN)); + + } + + }// for all other predicates + } + + private boolean checkAddValidViolationConstraint(String modifier, + InferenceVariableLexManager variables, int otherPredicateId, int otherCandidateId, + String otherLemma, int label, Set vars) { + + Map> vv; + if (manager.getSRLType() == SRLType.Verb) + vv = verbViolations; + else + vv = nomViolations; + + if (!vv.containsKey(modifier)) + return false; + + if (!vv.get(modifier).containsKey(otherLemma)) + return false; + + String arg = vv.get(modifier).get(otherLemma); + + return checkAddConstraint(variables, otherPredicateId, otherCandidateId, label, vars, arg); + + } + + /** + * Check if the other candidate has label 'arg'. If so, then there is no violation. So return + * true. Otherwise, add the variable to the accumulator set. + */ + private boolean checkAddConstraint(InferenceVariableLexManager variables, int otherPredicateId, + int otherCandidateId, int label, Set vars, String arg) { + boolean valid = false; + if (label >= 0) { + if (label == manager.getArgumentId(arg)) + valid = true; + } else { + int variable = + this.getArgumentVariable(variables, manager.getPredictedViewName(), + otherPredicateId, otherCandidateId, arg); + if (variable >= 0) + vars.add(variable); + } + return valid; + } + + private static Map> loadValidViolations(SRLType srlType) + throws Exception { + Map> map = new HashMap<>(); + + String file = srlType + ".cross-predicate.modifiers"; + + List list = IOUtils.lsResources(LegalArguments.class, file); + if (list.size() == 0) { + log.error("Cannot find file " + file + " in the classpath!"); + throw new Exception("Cannot find file " + file + " in the classpath!"); + } else { + + URL url = list.get(0); + Scanner scanner = new Scanner(url.openStream()); + + log.info("Loading allowed cross-argument" + " modifier arguments from {}", file); + while (scanner.hasNextLine()) { + String line = scanner.nextLine().trim(); + if (line.length() == 0) + continue; + + String[] parts = line.split("\t"); + String mod = parts[0]; + + map.put(mod, new HashMap()); + for (int i = 1; i < parts.length; i++) { + String[] pp = parts[i].split(","); + + String lemma = pp[0]; + String arg = pp[1]; + + map.get(mod).put(lemma, arg); + } + } + } + + return map; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/NoDuplicateCoreConstraint.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/NoDuplicateCoreConstraint.java new file mode 100644 index 000000000..b12aa75dc --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/NoDuplicateCoreConstraint.java @@ -0,0 +1,142 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference.constraints; + +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraint; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.core.SRLType; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPConstraintGenerator; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateStructure; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceStructure; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * This says that some arguments can't be repeated for a predicate. Originally, it was just used for + * core arguments, but it turns out that it applies to others too. + * + * @author Vivek Srikumar + * + */ +public class NoDuplicateCoreConstraint extends SRLILPConstraintGenerator { + + public final static String name = "noDuplicateCore"; + private final Set argSet; + + public NoDuplicateCoreConstraint(SRLManager manager) { + super(manager, name, true); + + argSet = new HashSet<>(manager.getCoreArguments()); + + // XXX: This might be a bit dubious. + boolean nom = manager.getSRLType() == SRLType.Nom; + if (nom) + argSet.add("SUP"); + + // XXX: These come from statistics on the training set. The following + // hold more than 99% of the time in the training set. + + if (nom) { + // 100 % + argSet.add("AM-ADV"); + argSet.add("AM-DIR"); + argSet.add("AM-DIS"); + argSet.add("AM-EXT"); + + // following are not always true, but very often + argSet.add("AM-LOC"); + argSet.add("AM-MNR"); + + } else { + argSet.add("AM-REC"); + } + + argSet.add("AM-NEG"); + argSet.add("AM-PNC"); + argSet.add("AM-PRD"); + + } + + @Override + public List getILPConstraints(IInstance x, InferenceVariableLexManager variables) { + return getViolatedILPConstraints(x, null, variables); + } + + private List addPredicateConstraints(SRLManager manager, + InferenceVariableLexManager variables, Set coreArgs, int predicateId, + SRLPredicateInstance x, SRLPredicateStructure y) { + + String type = manager.getPredictedViewName(); + int numCandidates = x.getCandidateInstances().size(); + + List list = new ArrayList<>(); + + for (String coreArgument : coreArgs) { + int count = 0; + + int argId = manager.getArgumentId(coreArgument); + + int[] vars = new int[numCandidates]; + double[] coefs = new double[numCandidates]; + for (int candidateId = 0; candidateId < numCandidates; candidateId++) { + vars[candidateId] = + getArgumentVariable(variables, type, predicateId, candidateId, coreArgument); + + coefs[candidateId] = 1; + + if (y != null) { + if (y.getArgLabel(candidateId) == argId) + count++; + } + } + + // XXX count == 0 doesn't seem right + if (y != null && count < 2) + continue; + + Pair cleanedVar = cleanupVariables(vars, coefs); + vars = cleanedVar.getFirst(); + coefs = cleanedVar.getSecond(); + + if (vars.length > 0) + list.add(new ILPConstraint(vars, coefs, 1.0, ILPConstraint.LESS_THAN)); + + } + + return list; + } + + @Override + public List getViolatedILPConstraints(IInstance ins, IStructure s, + InferenceVariableLexManager variables) { + + SRLSentenceInstance x = (SRLSentenceInstance) ins; + SRLSentenceStructure y = (SRLSentenceStructure) s; + + List list = new ArrayList<>(); + + for (int predicateId = 0; predicateId < x.numPredicates(); predicateId++) { + SRLPredicateInstance xp = x.predicates.get(predicateId); + + SRLPredicateStructure yp = y == null ? null : y.ys.get(predicateId); + + list.addAll(addPredicateConstraints(manager, variables, argSet, predicateId, xp, yp)); + } + + return list; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/NoOverlapConstraint.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/NoOverlapConstraint.java new file mode 100644 index 000000000..104acfb2b --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/NoOverlapConstraint.java @@ -0,0 +1,130 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference.constraints; + +import edu.illinois.cs.cogcomp.core.datastructures.IntPair; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraint; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPConstraintGenerator; +import edu.illinois.cs.cogcomp.srl.learn.*; + +import java.util.*; + +public class NoOverlapConstraint extends SRLILPConstraintGenerator { + + public static String name = "noOverlappingArguments"; + + public NoOverlapConstraint(SRLManager manager) { + super(manager, name, false); + } + + @Override + public List getILPConstraints(IInstance x, InferenceVariableLexManager variables) { + return getViolatedILPConstraints(x, null, variables); + } + + private List getNoOverlapConstraint(SRLManager manager, + InferenceVariableLexManager variables, int predicateId, SRLPredicateInstance x, + SRLPredicateStructure y) { + + String type = manager.getPredictedViewName(); + + final Map> wordToCandidateId = new HashMap<>(); + + List candidates = x.getCandidateInstances(); + + for (int candidateId = 0; candidateId < candidates.size(); candidateId++) { + + IntPair span = candidates.get(candidateId).getSpan(); + + for (int wordId = span.getFirst(); wordId < span.getSecond(); wordId++) { + if (!wordToCandidateId.containsKey(wordId)) + wordToCandidateId.put(wordId, new HashSet()); + wordToCandidateId.get(wordId).add(candidateId); + } + + } + + Set> overlaps = new HashSet<>(); + for (int wordId : wordToCandidateId.keySet()) { + overlaps.add(wordToCandidateId.get(wordId)); + } + + // int count = 0; + + int nullId = manager.getArgumentId(SRLManager.NULL_LABEL); + + List list = new ArrayList<>(); + + for (Set cands : overlaps) { + if (cands.size() == 1) + continue; + + int[] vars = new int[cands.size()]; + double[] coefs = new double[cands.size()]; + + int i = 0; + + int numNonNull = 0; + + for (int candId : cands) { + coefs[i] = 1; + vars[i] = + getArgumentVariable(variables, type, predicateId, candId, + SRLManager.NULL_LABEL); + i++; + + if (y != null) { + if (y.getArgLabel(candId) != nullId) + numNonNull++; + } + } + + if (y != null && numNonNull <= 1) + continue; + + Pair cleanedVar = cleanupVariables(vars, coefs); + vars = cleanedVar.getFirst(); + coefs = cleanedVar.getSecond(); + + if (vars.length > 0) { + // addGreaterThanConstraint(xmp, vars, coefs, vars.length - 1); + list.add(new ILPConstraint(vars, coefs, vars.length - 1, ILPConstraint.GREATER_THAN)); + // count++; + } + + } + + return list; + + } + + @Override + public List getViolatedILPConstraints(IInstance ins, IStructure s, + InferenceVariableLexManager variables) { + + SRLSentenceInstance x = (SRLSentenceInstance) ins; + SRLSentenceStructure y = (SRLSentenceStructure) s; + + List list = new ArrayList<>(); + + for (int predicateId = 0; predicateId < x.numPredicates(); predicateId++) { + SRLPredicateInstance xp = x.predicates.get(predicateId); + SRLPredicateStructure yp = y == null ? null : y.ys.get(predicateId); + + list.addAll(getNoOverlapConstraint(manager, variables, predicateId, xp, yp)); + } + + return list; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/PredicateSenseConstraints.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/PredicateSenseConstraints.java new file mode 100644 index 000000000..bcf3964f9 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/PredicateSenseConstraints.java @@ -0,0 +1,149 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference.constraints; + +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraint; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.core.SRLType; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPConstraintGenerator; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPInference; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateStructure; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceStructure; + +import java.util.*; + + +public class PredicateSenseConstraints extends SRLILPConstraintGenerator { + + public static final String name = "predicateSense"; + + public PredicateSenseConstraints(SRLManager manager) { + super(manager, name, true); + } + + @Override + public List getILPConstraints(IInstance x, InferenceVariableLexManager variables) { + return getViolatedILPConstraints(x, null, variables); + } + + @Override + public List getViolatedILPConstraints(IInstance ins, IStructure s, + InferenceVariableLexManager variables) { + + SRLSentenceInstance x = (SRLSentenceInstance) ins; + SRLSentenceStructure y = (SRLSentenceStructure) s; + + List list = new ArrayList<>(); + for (int predicateId = 0; predicateId < x.numPredicates(); predicateId++) { + + SRLPredicateInstance xp = x.predicates.get(predicateId); + SRLPredicateStructure yp = y == null ? null : y.ys.get(predicateId); + + list.addAll(getPredicateSenseConstraints(manager, predicateId, xp, yp, variables)); + } + + return list; + } + + private List getPredicateSenseConstraints(SRLManager manager, int predicateId, + SRLPredicateInstance x, SRLPredicateStructure y, InferenceVariableLexManager variables) { + List list = new ArrayList<>(); + + int numCandidates = x.getCandidateInstances().size(); + String type = manager.getPredictedViewName(); + + String lemma = x.getSenseInstance().getPredicateLemma(); + + Map> validLabelsForSense = manager.getLegalLabelsForSense(lemma); + + Set allLabels = manager.getAllArguments(); + + Set validSenses = manager.getLegalSenses(lemma); + + for (int candidateId = 0; candidateId < numCandidates; candidateId++) { + for (String sense : validSenses) { + if (y != null) { + if (y.getSense() != manager.getSenseId(sense)) + continue; + } + + String senseVarName = + SRLILPInference.getSenseVariableIdentifier(type, predicateId, sense); + int senseVar = variables.getVariable(senseVarName); + + Set validLabels = validLabelsForSense.get(sense); + + Set labels = new HashSet<>(); + + for (String label : allLabels) { + if (label.startsWith("AM-") || label.startsWith("R-AM-") + || label.startsWith("C-AM-")) + labels.add(label); + else { + String ll = label.replaceAll("R-", "").replaceAll("C-", ""); + if (validLabels.contains(ll)) + labels.add(label); + } + } + + labels.add("C-V"); + labels.add(SRLManager.NULL_LABEL); + if (manager.getSRLType() == SRLType.Nom) { + labels.add("SUP"); + labels.add("C-SUP"); + labels.add("A1"); + } + + if (y != null) { + int argLabel = y.getArgLabel(candidateId); + if (labels.contains(manager.getArgument(argLabel))) + continue; + } + + int[] vars = new int[labels.size() + 2]; + double[] coefs = new double[vars.length]; + vars[0] = senseVar; + coefs[0] = -1; + + vars[1] = + getArgumentVariable(variables, type, predicateId, candidateId, + SRLManager.NULL_LABEL); + coefs[1] = 1.0; + + int i = 2; + for (String validLabel : labels) { + int var = + getArgumentVariable(variables, type, predicateId, candidateId, + validLabel); + vars[i] = var; + coefs[i] = 1.0; + i++; + } + + Pair cleanedVar = cleanupVariables(vars, coefs); + vars = cleanedVar.getFirst(); + coefs = cleanedVar.getSecond(); + + if (vars.length > 0) { + // addGreaterThanConstraint(xmp, vars, coefs, 0); + list.add(new ILPConstraint(vars, coefs, 0, ILPConstraint.GREATER_THAN)); + } + + }// foreach valid sense + + }// foreach candidate + + return list; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/RArgConstraints.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/RArgConstraints.java new file mode 100644 index 000000000..efa8590d8 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/RArgConstraints.java @@ -0,0 +1,148 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference.constraints; + +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraint; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPConstraintGenerator; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateStructure; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceStructure; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +public class RArgConstraints extends SRLILPConstraintGenerator { + + private final static Logger log = LoggerFactory.getLogger(RArgConstraints.class); + + public final static String name = "RArgumentConstraint"; + + public RArgConstraints(SRLManager manager) { + super(manager, name, true); + } + + @Override + public List getILPConstraints(IInstance x, InferenceVariableLexManager variables) { + + log.debug("Adding R-Arg constraints"); + return getViolatedILPConstraints(x, null, variables); + } + + @Override + public List getViolatedILPConstraints(IInstance ins, IStructure s, + InferenceVariableLexManager variables) { + + SRLSentenceInstance x = (SRLSentenceInstance) ins; + SRLSentenceStructure y = (SRLSentenceStructure) s; + + List list = new ArrayList<>(); + + for (int predicateId = 0; predicateId < x.numPredicates(); predicateId++) { + + SRLPredicateInstance xp = x.predicates.get(predicateId); + SRLPredicateStructure yp = y == null ? null : y.ys.get(predicateId); + + log.debug("Predicate: " + x); + list.addAll(addPredicateRArgConstraints(manager, variables, predicateId, xp, yp)); + } + + return list; + } + + private List addPredicateRArgConstraints(SRLManager manager, + InferenceVariableLexManager variables, int predicateId, SRLPredicateInstance x, + SRLPredicateStructure y) { + + Set coreArgs = manager.getCoreArguments(); + + List list = new ArrayList<>(); + for (String arg : coreArgs) { + list.addAll(addRArgConstraint(manager, predicateId, variables, x, y, arg)); + } + + return list; + } + + private List addRArgConstraint(SRLManager manager, int predicateId, + InferenceVariableLexManager variables, SRLPredicateInstance x, SRLPredicateStructure y, + String arg) { + + String type = manager.getPredictedViewName(); + int numCandidates = x.getCandidateInstances().size(); + + String rArg = "R-" + arg; + + log.debug("R-Arg constraint for {} for predicate {}", rArg, x); + + List list = new ArrayList<>(); + for (int otherCandidateId = 0; otherCandidateId < numCandidates; otherCandidateId++) { + + if (y != null) { + if (y.getArgLabel(otherCandidateId) != manager.getArgumentId(rArg)) + continue; + } + + int[] vars = new int[numCandidates]; + double[] coefs = new double[numCandidates]; + + int rArgVariable = + getArgumentVariable(variables, type, predicateId, otherCandidateId, rArg); + if (rArgVariable < 0) + continue; + + vars[0] = rArgVariable; + coefs[0] = -1; + + int id = 1; + + boolean foundArg = false; + + for (int candidateId = 0; candidateId < numCandidates; candidateId++) { + if (candidateId == otherCandidateId) + continue; + + int variable = getArgumentVariable(variables, type, predicateId, candidateId, arg); + + vars[id] = variable; + coefs[id] = 1; + id++; + + if (y != null) + if (y.getArgLabel(candidateId) == manager.getArgumentId(arg)) + foundArg = true; + } + + if (y != null && foundArg) + continue; + + Pair pair = cleanupVariables(vars, coefs); + + vars = pair.getFirst(); + coefs = pair.getSecond(); + + ILPConstraint ilpConstraint = + new ILPConstraint(vars, coefs, 0, ILPConstraint.GREATER_THAN); + + log.debug("Constraint: " + ilpConstraint); + list.add(ilpConstraint); + + } + + return list; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/SupportVerbConstraint.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/SupportVerbConstraint.java new file mode 100644 index 000000000..ab8c11e40 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/inference/constraints/SupportVerbConstraint.java @@ -0,0 +1,107 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.inference.constraints; + +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.edison.features.helpers.WordHelpers; +import edu.illinois.cs.cogcomp.infer.ilp.ILPConstraint; +import edu.illinois.cs.cogcomp.infer.ilp.InferenceVariableLexManager; +import edu.illinois.cs.cogcomp.nlp.utilities.POSUtils; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.inference.SRLILPConstraintGenerator; +import edu.illinois.cs.cogcomp.srl.learn.SRLPredicateInstance; +import edu.illinois.cs.cogcomp.srl.learn.SRLSentenceInstance; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * This constraint says that a label can be SUP (or C-SUP) ONLY if it is either a verb or a Nominal + * predicate. We assume that the predicates are fixed. + * + * @author svivek + */ +public class SupportVerbConstraint extends SRLILPConstraintGenerator { + + public static final String name = "supportVerbConstraint"; + + public SupportVerbConstraint(SRLManager manager) { + super(manager, name, false); + } + + @Override + public List getILPConstraints(IInstance ins, + InferenceVariableLexManager variables) { + + SRLSentenceInstance x = (SRLSentenceInstance) ins; + + List list = new ArrayList<>(); + + Set predicatePosition = new HashSet<>(); + for (int predicateId = 0; predicateId < x.numPredicates(); predicateId++) { + predicatePosition.add(x.predicates.get(predicateId).getSenseInstance().getSpan() + .getFirst()); + } + + for (int predicateId = 0; predicateId < x.numPredicates(); predicateId++) { + SRLPredicateInstance xp = x.predicates.get(predicateId); + TextAnnotation ta = xp.getSenseInstance().getConstituent().getTextAnnotation(); + + int predicate = xp.getSenseInstance().getSpan().getFirst(); + for (int candidateId = 0; candidateId < xp.getCandidateInstances().size(); candidateId++) { + + int candidatePosition = + xp.getCandidateInstances().get(candidateId).getSpan().getFirst(); + + String pos = WordHelpers.getPOS(ta, candidatePosition); + if (POSUtils.isPOSVerb(pos)) + continue; + + if (predicatePosition.contains(candidatePosition) && candidatePosition != predicate) + continue; + + int supportVar = + getArgumentVariable(variables, manager.getPredictedViewName(), predicateId, + candidateId, "SUP"); + + // if there is no support, then there cannot be C-support, so + // don't bother. + if (supportVar < 0) + continue; + + int cSupportVar = + getArgumentVariable(variables, manager.getPredictedViewName(), predicateId, + candidateId, "C-SUP"); + + if (cSupportVar >= 0) { + + list.add(new ILPConstraint(new int[] {supportVar, cSupportVar}, new double[] { + 1, 1}, 0, ILPConstraint.EQUAL)); + } else { + list.add(new ILPConstraint(new int[] {supportVar}, new double[] {1}, 0, + ILPConstraint.EQUAL)); + } + + } + } + + return list; + } + + @Override + public List getViolatedILPConstraints(IInstance x, IStructure y, + InferenceVariableLexManager variables) { + // This constraint is always present + return new ArrayList<>(); + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/IdentifierThresholdTuner.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/IdentifierThresholdTuner.java new file mode 100644 index 000000000..cf66fa09e --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/IdentifierThresholdTuner.java @@ -0,0 +1,208 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.learn; + +import edu.illinois.cs.cogcomp.core.datastructures.IntPair; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.math.Permutations; +import edu.illinois.cs.cogcomp.core.utilities.StringUtils; +import edu.illinois.cs.cogcomp.sl.core.SLProblem; +import edu.illinois.cs.cogcomp.srl.core.ArgumentIdentifier; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; + +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.*; + +public class IdentifierThresholdTuner { + + private final double n_F; + private final int nThreads; + private final SRLManager manager; + private final SLProblem problem; + + public IdentifierThresholdTuner(SRLManager manager, double N_F, SLProblem problem) { + this.manager = manager; + + this.n_F = N_F; + this.problem = problem; + + this.nThreads = Math.min(8, Runtime.getRuntime().availableProcessors()); + + } + + private double fN(double precision, double recall, double n) { + double denom = n * n * precision + recall; + double num = (n * n + 1) * precision * recall; + + if (denom == 0) + return 0; + + return num / denom; + + } + + public Pair tuneIdentifierScale(List A, List B) + throws Exception { + + List> scores = new ArrayList<>(); + + int totalGold = 0; + int numExamples = 0; + ArgumentIdentifier rawScorer = new ArgumentIdentifier(1.0, 0.0, manager); + + for (int i = 0; i < this.problem.size(); i++) { + SRLMulticlassInstance x = (SRLMulticlassInstance) this.problem.instanceList.get(i); + SRLMulticlassLabel y = (SRLMulticlassLabel) this.problem.goldStructureList.get(i); + + assert y.getLabel() == 0 || y.getLabel() == 1; + + boolean goldLabel = y.getLabel() == 1; + + if (goldLabel) + totalGold++; + + double rawScore = rawScorer.getIdentifierRawScore(x); + + scores.add(new Pair<>(rawScore, goldLabel)); + numExamples++; + if (numExamples % 10000 == 0) { + System.out.println(numExamples + " scores cached"); + } + } + + Map, IntPair> perf = getPerformance(A, B, scores); + + List> list = new ArrayList<>(); + + double maxF = Double.NEGATIVE_INFINITY; + Pair maxer = null; + + for (Entry, IntPair> entry : perf.entrySet()) { + Pair key = entry.getKey(); + IntPair value = entry.getValue(); + + double totalPredicted = value.getFirst(); + double correct = value.getSecond(); + + double precision = 0, recall = 0, f; + + if (totalPredicted > 0) + precision = correct / totalPredicted; + + if (totalGold > 0) + recall = correct / totalGold; + + f = fN(precision, recall, n_F); + + String output = key.toString(); + output += "\t" + totalGold; + output += "\t" + (int) (totalPredicted); + output += "\t" + (int) (correct); + + output += "\t" + StringUtils.getFormattedTwoDecimal(precision * 100); + output += "\t" + StringUtils.getFormattedTwoDecimal(recall * 100); + output += "\t" + StringUtils.getFormattedTwoDecimal(f * 100); + + list.add(new Pair<>(output, f)); + + if (f > maxF) { + maxF = f; + maxer = key; + } + } + + System.out.println(); + System.out.println("Based on F" + n_F + " measure, recommended (A, B) = " + maxer); + + System.out.println("Top 100 values"); + + System.out.println("(A, B)\ttotalGold\ttotalPredicted\tcorrect\tP\tR\tF" + n_F); + + Collections.sort(list, new Comparator>() { + + @Override + public int compare(Pair arg0, Pair arg1) { + return -arg0.getSecond().compareTo(arg1.getSecond()); + } + }); + + for (int i = 0; i < 100; i++) { + System.out.println(list.get(i).getFirst()); + } + + return maxer; + + } + + @SuppressWarnings("unchecked") + private Map, IntPair> getPerformance(List A, List B, + final List> scores) throws InterruptedException, + ExecutionException { + + ExecutorService executor = Executors.newFixedThreadPool(nThreads); + + List, IntPair>>> tasks = new ArrayList<>(); + + for (List element : Permutations.crossProduct(Arrays.asList(A, B))) { + + final double a = element.get(0); + final double b = element.get(1); + FutureTask, IntPair>> task = + new FutureTask<>(new Callable, IntPair>>() { + + @Override + public Pair, IntPair> call() throws Exception { + return getPerformance(a, b, scores); + } + }); + + tasks.add(task); + executor.execute(task); + + } + + executor.shutdown(); + + Map, IntPair> map = new HashMap<>(); + for (FutureTask, IntPair>> task : tasks) { + Pair, IntPair> out = task.get(); + map.put(out.getFirst(), out.getSecond()); + } + + return map; + } + + private Pair, IntPair> getPerformance(double A, double B, + List> scores) { + + ArgumentIdentifier identifier = new ArgumentIdentifier(A, B, manager); + + int totalPredicted = 0; + int totalCorrectTrue = 0; + for (Pair entry : scores) { + double rawScore = entry.getFirst(); + boolean goldLabel = entry.getSecond(); + + double scaledScore = identifier.scaleIdentifierScore(rawScore); + + boolean prediction = scaledScore >= 0; + + if (prediction) { + totalPredicted++; + if (goldLabel) + totalCorrectTrue++; + } + + } + + IntPair perf = new IntPair(totalPredicted, totalCorrectTrue); + + return new Pair<>(new Pair<>(A, B), perf); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLFeatureExtractor.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLFeatureExtractor.java new file mode 100644 index 000000000..bb59000e3 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLFeatureExtractor.java @@ -0,0 +1,42 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.learn; + +import edu.illinois.cs.cogcomp.sl.core.AbstractFeatureGenerator; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.sl.util.FeatureVectorBuffer; +import edu.illinois.cs.cogcomp.sl.util.IFeatureVector; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; +import edu.illinois.cs.cogcomp.srl.experiment.PreExtractor; + +/** + * A wrapper for the feature extractor required by illinois-sl. The real feature extraction happens + * in {@link PreExtractor#consumeInstance(SRLMulticlassInstance, SRLMulticlassLabel)} during + * training and {@link SRLPredicateInstance#cacheAllFeatureVectors(boolean)} during testing. + * + * @author upadhya3 + */ +public class SRLFeatureExtractor extends AbstractFeatureGenerator { + + public SRLFeatureExtractor() {} + + @Override + public IFeatureVector getFeatureVector(IInstance x, IStructure y) { + SRLMulticlassInstance mi = (SRLMulticlassInstance) x; + SRLMulticlassLabel my = (SRLMulticlassLabel) y; + int label = my.getLabel(); + SRLManager manager = my.getManager(); + Models type = my.getType(); + IFeatureVector fv = mi.getCachedFeatureVector(type); + FeatureVectorBuffer fvb = new FeatureVectorBuffer(fv); + fvb.shift(label * manager.getModelInfo(type).getLexicon().size()); + return fvb.toFeatureVector(); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLMulticlassInstance.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLMulticlassInstance.java new file mode 100644 index 000000000..4c447a751 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLMulticlassInstance.java @@ -0,0 +1,117 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.learn; + +import edu.illinois.cs.cogcomp.core.datastructures.IntPair; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.edison.features.Feature; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.sl.util.IFeatureVector; +import edu.illinois.cs.cogcomp.sl.util.SparseFeatureVector; +import edu.illinois.cs.cogcomp.srl.core.ModelInfo; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public class SRLMulticlassInstance implements IInstance { + + private final Constituent c; + + private final Map features; + private String predicateLemma; + + private final Constituent predicate; + + private SRLManager manager; + + public SRLMulticlassInstance(Constituent c, Constituent predicate, SRLManager manager) { + this.c = c; + this.predicate = predicate; + this.manager = manager; + predicateLemma = predicate.getAttribute(PredicateArgumentView.LemmaIdentifier); + + features = new ConcurrentHashMap<>(); + } + + public SRLMulticlassInstance(Models model, String lemma, String features) { + c = null; + this.predicate = null; + this.predicateLemma = lemma; + + this.features = new ConcurrentHashMap<>(); + + this.cacheFeatureVector(model, getFeatureVector(features)); + } + + public double size() { + return 1; + } + + public String getPredicateLemma() { + return predicateLemma; + } + + @Override + public String toString() { + return "SRLMulticlassInstance [cand=" + c + ", predicate=" + predicateLemma + "]"; + } + + public void cacheFeatureVector(Models m, IFeatureVector f) { + assert !features.containsKey(m); + features.put(m, f); + } + + public IFeatureVector getCachedFeatureVector(Models m) { + assert features.containsKey(m); + return features.get(m); + } + + private IFeatureVector getFeatureVector(String features) { + String[] parts = features.split(" "); + int[] idx = new int[parts.length]; + float[] vals = new float[parts.length]; + + for (int i = 0; i < parts.length; i++) { + String[] f = parts[i].split(":"); + + idx[i] = Integer.parseInt(f[0]); + vals[i] = Float.parseFloat(f[1]); + } + + return new SparseFeatureVector(idx, vals); + } + + public Constituent getConstituent() { + return c; + } + + public IntPair getSpan() { + return c.getSpan(); + } + + public Constituent getPredicate() { + return predicate; + } + + public void cacheFeatureVector(Models model, Set features) { + Map featureMap = new HashMap<>(); + for (Feature f : features) { + featureMap.put(f.getName(), f.getValue()); + } + + ModelInfo modelInfo = manager.getModelInfo(model); + Pair feats = modelInfo.getLexicon().getFeatureVector(featureMap); + this.cacheFeatureVector(model, new SparseFeatureVector(feats.getFirst(), feats.getSecond())); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLMulticlassLabel.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLMulticlassLabel.java new file mode 100644 index 000000000..a9e85b408 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLMulticlassLabel.java @@ -0,0 +1,37 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.learn; + +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; + +public class SRLMulticlassLabel implements IStructure { + private int label; + private Models type; + private SRLManager manager; + + public SRLMulticlassLabel(int label, Models type, SRLManager manager) { + this.label = label; + this.type = type; + this.manager = manager; + } + + public Models getType() { + return type; + } + + public SRLManager getManager() { + return manager; + } + + public int getLabel() { + return label; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLPredicateInstance.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLPredicateInstance.java new file mode 100644 index 000000000..1c2448527 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLPredicateInstance.java @@ -0,0 +1,183 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.learn; + +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.utilities.Parallel; +import edu.illinois.cs.cogcomp.edison.features.Feature; +import edu.illinois.cs.cogcomp.sl.core.IInstance; +import edu.illinois.cs.cogcomp.srl.SRLProperties; +import edu.illinois.cs.cogcomp.srl.core.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +public class SRLPredicateInstance implements IInstance { + private static int FEATURE_EXTRACTION_N_THREADS; + + private final static Logger log = LoggerFactory.getLogger(SRLPredicateInstance.class); + + private final List candidates; + private final SRLMulticlassInstance senseInstance; + private final SRLManager manager; + + public SRLPredicateInstance(List candidates, + SRLMulticlassInstance sense, SRLManager manager) { + this.candidates = candidates; + this.senseInstance = sense; + this.manager = manager; + } + + public SRLPredicateInstance(Constituent predicate, SRLManager manager) { + this.manager = manager; + + // XXX Generate a clone of the predicate to avoid changing the gold TA + Constituent predicateClone = predicate.cloneForNewView(predicate.getViewName()); + + senseInstance = new SRLMulticlassInstance(predicateClone, predicateClone, manager); + + List list = new ArrayList<>(); + + ArgumentCandidateGenerator candidateGenerator = manager.getArgumentCandidateGenerator(); + + List cands = candidateGenerator.generateCandidates(predicate); + + for (Constituent c : cands) { + list.add(new SRLMulticlassInstance(c, predicateClone, manager)); + } + this.candidates = Collections.unmodifiableList(list); + } + + public SRLPredicateInstance(Constituent predicate, SRLManager manager, + ArgumentIdentifier identifier) { + this.manager = manager; + + // XXX Generate a clone of the predicate to avoid changing the gold TA + Constituent predicateClone = predicate.cloneForNewView(predicate.getViewName()); + + senseInstance = new SRLMulticlassInstance(predicateClone, predicateClone, manager); + + ArgumentCandidateGenerator candidateGenerator = manager.getArgumentCandidateGenerator(); + + List allCandidates = new ArrayList<>(); + + for (Constituent c : candidateGenerator.generateCandidates(predicate)) { + allCandidates.add(new SRLMulticlassInstance(c, predicateClone, manager)); + } + + cacheIdentifierFeatures(allCandidates); + + List list = new ArrayList<>(); + + for (SRLMulticlassInstance c : allCandidates) { + if (identifier.getIdentifierScaledDecision(c)) + list.add(c); + } + + this.candidates = Collections.unmodifiableList(list); + } + + public List getCandidateInstances() { + return candidates; + } + + public SRLMulticlassInstance getSenseInstance() { + return senseInstance; + } + + public double size() { + return this.candidates.size() + 1; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + for (SRLMulticlassInstance c : candidates) { + sb.append("Candidate: ").append(c.toString()).append("\n"); + } + + sb.append("Sense: ").append(senseInstance.toString()).append("\n"); + + return sb.toString(); + } + + private synchronized void cacheIdentifierFeatures(List xs) { + for (SRLMulticlassInstance x : xs) { + ModelInfo modelInfo = manager.getModelInfo(Models.Identifier); + + try { + Set feats = modelInfo.fex.getFeatures(x.getConstituent()); + + x.cacheFeatureVector(Models.Identifier, feats); + } catch (Exception e) { + log.error("Unable to extract features for {}", x, e); + + throw new RuntimeException(e); + } + } + } + + /** + * This method caches features for all argument candidates for given predicate. This is used + * only during testing to speed up prediction. + * + * @param cacheIdentifier Should the identifier also be cached? + */ + public void cacheAllFeatureVectors(boolean cacheIdentifier) { + List> list = new ArrayList<>(); + list.add(new Pair<>(senseInstance, Models.Sense)); + for (SRLMulticlassInstance x : this.getCandidateInstances()) { + list.add(new Pair<>(x, Models.Classifier)); + + if (cacheIdentifier) + list.add(new Pair<>(x, Models.Identifier)); + } + + Parallel.Method> function = + new Parallel.Method>() { + + @Override + public void run(Pair argument) { + SRLMulticlassInstance x = argument.getFirst(); + Models m = argument.getSecond(); + + ModelInfo modelInfo = manager.getModelInfo(m); + + try { + Set feats = modelInfo.fex.getFeatures(x.getConstituent()); + x.cacheFeatureVector(m, feats); + } catch (Exception e) { + log.error("Unable to extract features for {}", x, e); + + throw new RuntimeException(e); + } + } + }; + SRLProperties props = SRLProperties.getInstance(); + FEATURE_EXTRACTION_N_THREADS = + Math.min(props.getNumFeatExtThreads(), Runtime.getRuntime().availableProcessors()); + + log.debug("Using {} threads for feature ext.", FEATURE_EXTRACTION_N_THREADS); + try { + int timeout = 10; + Parallel.forLoop(FEATURE_EXTRACTION_N_THREADS, list, function, timeout, + TimeUnit.MINUTES); + } catch (Exception e) { + log.error("Waited for ten minutes for feature extraction. Giving up!", e); + } + + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLPredicateStructure.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLPredicateStructure.java new file mode 100644 index 000000000..bb114229f --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLPredicateStructure.java @@ -0,0 +1,73 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.learn; + +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.Models; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; + +public class SRLPredicateStructure implements IStructure { + + public final SRLPredicateInstance x; + private final int[] argLabels; + private final int sense; + private final SRLManager manager; + + public SRLPredicateStructure(SRLPredicateInstance x, int[] argLabels, int sense, + SRLManager manager) { + this.x = x; + this.argLabels = argLabels; + this.sense = sense; + this.manager = manager; + + } + + // @Override + // public FeatureVector getFeatureVector() { + // throw new RuntimeException("Not yet implemented!"); + // } + + public int getArgLabel(int candidateId) { + return argLabels[candidateId]; + } + + public int getSense() { + return sense; + } + + public SRLMulticlassLabel getClassifierMulticlassLabel(int candidateId) { + return new SRLMulticlassLabel(argLabels[candidateId], Models.Classifier, manager); + } + + public SRLMulticlassLabel getIdentifierMulticlassLabel(int candidateId) { + boolean isNull = manager.isNullLabel(argLabels[candidateId]); + + int c = isNull ? 0 : 1; + + return new SRLMulticlassLabel(c, Models.Identifier, manager); + } + + public SRLMulticlassLabel getSenseMulticlassLabel() { + return new SRLMulticlassLabel(sense, Models.Sense, manager); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + for (int candidateId = 0; candidateId < this.argLabels.length; candidateId++) { + sb.append("Candidate ").append(candidateId).append(": ") + .append(manager.getArgument(argLabels[candidateId])).append("\n"); + } + + sb.append("Sense: ").append(manager.getSense(sense)).append("\n"); + + return sb.toString(); + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLSentenceInstance.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLSentenceInstance.java new file mode 100644 index 000000000..bdb10eed0 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLSentenceInstance.java @@ -0,0 +1,34 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.learn; + +import edu.illinois.cs.cogcomp.sl.core.IInstance; + +import java.util.List; + +public class SRLSentenceInstance implements IInstance { + + public final List predicates; + private int size; + + public SRLSentenceInstance(List instances) { + this.predicates = instances; + size = 0; + for (SRLPredicateInstance x : instances) { + size += x.size(); + } + } + + public double size() { + return size; + } + + public int numPredicates() { + return predicates.size(); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLSentenceStructure.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLSentenceStructure.java new file mode 100644 index 000000000..89487bc35 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/learn/SRLSentenceStructure.java @@ -0,0 +1,78 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.learn; + +import edu.illinois.cs.cogcomp.core.datastructures.IntPair; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.sl.core.IStructure; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; + +import java.util.ArrayList; +import java.util.List; + +public class SRLSentenceStructure implements IStructure { + + public final SRLSentenceInstance x; + public final List ys; + + public SRLSentenceStructure(SRLSentenceInstance instance, List ys) { + this.x = instance; + this.ys = ys; + } + + public PredicateArgumentView getView(SRLManager manager, TextAnnotation ta) { + String viewName = manager.getPredictedViewName(); + PredicateArgumentView pav = + new PredicateArgumentView(viewName, manager.getSRLSystemIdentifier(), ta, 1.0); + + int nullId = manager.getArgumentId(SRLManager.NULL_LABEL); + for (SRLPredicateStructure y : this.ys) { + SRLPredicateInstance x = y.x; + + SRLMulticlassInstance senseInstance = x.getSenseInstance(); + IntPair predicateSpan = senseInstance.getConstituent().getSpan(); + String predicateLemma = senseInstance.getPredicateLemma(); + + Constituent predicate = + new Constituent("Predicate", viewName, ta, predicateSpan.getFirst(), + predicateSpan.getSecond()); + predicate.addAttribute(PredicateArgumentView.LemmaIdentifier, predicateLemma); + + String sense = manager.getSense(y.getSense()); + predicate.addAttribute(PredicateArgumentView.SenseIdentifer, sense); + + List args = new ArrayList<>(); + List relations = new ArrayList<>(); + + List candidateInstances = x.getCandidateInstances(); + + for (int candidateId = 0; candidateId < candidateInstances.size(); candidateId++) { + if (y.getArgLabel(candidateId) == nullId) + continue; + + SRLMulticlassInstance ci = candidateInstances.get(candidateId); + IntPair span = ci.getConstituent().getSpan(); + + assert span.getFirst() <= span.getSecond() : ta; + + String label = manager.getArgument(y.getArgLabel(candidateId)); + + args.add(new Constituent(label, viewName, ta, span.getFirst(), span.getSecond())); + relations.add(label); + } + + pav.addPredicateArguments(predicate, args, + relations.toArray(new String[relations.size()]), new double[relations.size()]); + + } + + return pav; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/nom/NomArgumentCandidateGenerator.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/nom/NomArgumentCandidateGenerator.java new file mode 100644 index 000000000..3f09983a3 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/nom/NomArgumentCandidateGenerator.java @@ -0,0 +1,154 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.nom; + +import edu.illinois.cs.cogcomp.core.datastructures.IntPair; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.datastructures.trees.Tree; +import edu.illinois.cs.cogcomp.core.datastructures.trees.TreeTraversal; +import edu.illinois.cs.cogcomp.nlp.utilities.POSUtils; +import edu.illinois.cs.cogcomp.nlp.utilities.ParseTreeProperties; +import edu.illinois.cs.cogcomp.nlp.utilities.ParseUtils; +import edu.illinois.cs.cogcomp.srl.core.ArgumentCandidateGenerator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class NomArgumentCandidateGenerator extends ArgumentCandidateGenerator { + + private final static Logger log = LoggerFactory.getLogger(NomArgumentCandidateGenerator.class); + + public NomArgumentCandidateGenerator(NomSRLManager manager) { + super(manager); + } + + @Override + public String getCandidateViewName() { + return "NomArgumentCandidateView"; + } + + @Override + public List generateCandidates(Constituent predicate) { + TextAnnotation ta = predicate.getTextAnnotation(); + + int predicateSentenceId = ta.getSentenceId(predicate); + + int predicateSentenceStart = ta.getSentence(predicateSentenceId).getStartSpan(); + + // get the parse tree + Tree tree = ParseUtils.getParseTree(manager.defaultParser, ta, predicateSentenceId); + Tree> spanLabeledTree = ParseUtils.getSpanLabeledTree(tree); + + Constituent predicateClone = predicate.cloneForNewView(this.getCandidateViewName()); + + int predicatePosition = predicate.getStartSpan() - predicateSentenceStart; + + Set out = new HashSet<>(); + + // add all non terminals in the tree + for (Tree> c : TreeTraversal.depthFirstTraversal(spanLabeledTree)) { + if (!c.isRoot() && !c.isLeaf() && !c.getChild(0).isLeaf()) { + int start = c.getLabel().getSecond().getFirst() + predicateSentenceStart; + int end = c.getLabel().getSecond().getSecond() + predicateSentenceStart; + + out.add(getNewConstituent(ta, predicateClone, start, end)); + } + } + + Tree> predicateNode = + spanLabeledTree.getYield().get(predicatePosition); + // add all siblings of the predicate + for (Tree> sibling : predicateNode.getParent().getParent() + .getChildren()) { + Pair siblingNode = sibling.getLabel(); + IntPair siblingSpan = siblingNode.getSecond(); + int siblingSpanFirst = siblingSpan.getFirst(); + int siblingSpanSecond = siblingSpan.getSecond(); + + // unlike in VerbSRL (XuePalmerCandidateGenerator) we do allow the predicate + // to also be an argument, so there is no need to exclude it here + + int start = siblingSpanFirst + predicateSentenceStart; + int end = siblingSpanSecond + predicateSentenceStart; + + out.add(getNewConstituent(ta, predicateClone, start, end)); + } + + // verb nodes that dominate the predicate + Tree> node = predicateNode.getParent(); + + while (!node.isRoot() && !ParseTreeProperties.isNonTerminalVerb(node.getLabel().getFirst())) + node = node.getParent(); + + for (Tree> verbCandidate : node.getYield()) { + if (POSUtils.isPOSVerb(verbCandidate.getParent().getLabel().getFirst())) { + int start = + verbCandidate.getLabel().getSecond().getFirst() + predicateSentenceStart; + int end = start + 1; + + out.add(getNewConstituent(ta, predicateClone, start, end)); + } + } + + // pronouns in NPs within the same clause that dominate this predicate + node = predicateNode.getParent(); + while (!node.isRoot()) { + String label = node.getLabel().getFirst(); + if (label.startsWith("S")) + break; + + if (ParseTreeProperties.isNonTerminalNoun(label)) { + for (Tree> nominalCandidate : node.getYield()) { + if (POSUtils.isPOSPossessivePronoun(nominalCandidate.getParent().getLabel() + .getFirst())) { + int start = + nominalCandidate.getLabel().getSecond().getFirst() + + predicateSentenceStart; + int end = start + 1; + + out.add(getNewConstituent(ta, predicateClone, start, end)); + } + } + } + node = node.getParent(); + + } + + // if predicate is dominated by a PP, then the head of that PP + node = predicateNode.getParent(); + boolean ppParentFound = false; + while (!node.isRoot()) { + String label = node.getLabel().getFirst(); + if (ParseTreeProperties.isNonTerminalPP(label)) { + ppParentFound = true; + break; + } else if (label.startsWith("S")) + break; + + node = node.getParent(); + } + + if (ppParentFound) { + int start = node.getLabel().getSecond().getFirst() + predicateSentenceStart; + int end = start + 1; + out.add(getNewConstituent(ta, predicateClone, start, end)); + } + + log.debug("Number of candidates for {} from heuristic: {}", predicate.toString(), + out.toString()); + return new ArrayList<>(out); + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/nom/NomPredicateDetectorHeuristic.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/nom/NomPredicateDetectorHeuristic.java new file mode 100644 index 000000000..78047ddfa --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/nom/NomPredicateDetectorHeuristic.java @@ -0,0 +1,210 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.nom; + +import edu.illinois.cs.cogcomp.core.datastructures.Option; +import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TokenLabelView; +import edu.illinois.cs.cogcomp.edison.utilities.EdisonException; +import edu.illinois.cs.cogcomp.edison.utilities.NomLexEntry; +import edu.illinois.cs.cogcomp.edison.utilities.NomLexEntry.NomLexClasses; +import edu.illinois.cs.cogcomp.edison.utilities.NomLexReader; +import edu.illinois.cs.cogcomp.nlp.utilities.POSUtils; +import edu.illinois.cs.cogcomp.srl.core.AbstractPredicateDetector; +import org.tartarus.snowball.SnowballStemmer; +import org.tartarus.snowball.ext.englishStemmer; + +import java.util.*; + +public class NomPredicateDetectorHeuristic extends AbstractPredicateDetector { + + private static final SnowballStemmer stemmer = new englishStemmer(); + private final static Map nonStandard; + private final static Set pluralLemmas; + + private static final Set classes; + + static { + + nonStandard = new HashMap<>(); + + nonStandard.put("bondholder", "holder"); + nonStandard.put("earthquake", "quake"); + nonStandard.put("spokesperson", "person"); + nonStandard.put("allies", "ally"); + nonStandard.put("liabilities", "liability"); + nonStandard.put("reelection", "election"); + nonStandard.put("coauthor", "author"); + nonStandard.put("people", "person"); + nonStandard.put("supressor", "suppressor"); + nonStandard.put("buyout", "buy-out"); + nonStandard.put("hookup", "hook-up"); + nonStandard.put("ceasefire", "cease-fire"); + nonStandard.put("startup", "start-up"); + nonStandard.put("eurobond", "bond"); + + pluralLemmas = new HashSet<>(); + pluralLemmas.addAll(Arrays.asList("filers", "hundreds", "thousands", "millions", + "billions", "tens")); + + // All NomLex classes are accounted for here. + classes = new HashSet<>(); + classes.addAll(NomLexEntry.VERBAL); + classes.addAll(NomLexEntry.ADJECTIVAL); + classes.addAll(NomLexEntry.NON_VERB_ADJ); + + } + private final NomLexReader nomLex; + + public NomPredicateDetectorHeuristic(NomSRLManager manager) throws EdisonException { + super(manager); + nomLex = NomLexReader.getInstance(); + } + + @Override + public Option getLemma(TextAnnotation ta, int tokenId) { + + String pos = POSUtils.getPOS(ta, tokenId); + + boolean isNoun = POSUtils.isPOSNoun(pos); + if (!isNoun) { + return Option.empty(); + } else { + + Option opt; + String token = ta.getToken(tokenId).toLowerCase(); + if (pluralLemmas.contains(token)) { + opt = testTokenVariations(token); + } else { + + TokenLabelView lemmaView = (TokenLabelView) ta.getView(ViewNames.LEMMA); + String lemma = lemmaView.getConstituentAtToken(tokenId).getLabel(); + + opt = testTokenVariations(lemma); + + if (!opt.isPresent() && !lemma.matches("-*")) { + opt = testWithDelim(lemma, '-'); + } + + if (!opt.isPresent()) { + opt = testWithDelim(lemma, ' '); + } + } + + return opt; + } + } + + private Option testWithDelim(String token, char delim) { + Option found = Option.empty(); + + if (debug) { + System.out.println("Testing with delimiter: " + delim + ". Token=" + token); + } + + if (token.indexOf(delim) >= 0) { + String[] split = token.split("" + delim); + String lastElement = split[split.length - 1]; + + if (debug) { + + System.out.println(lastElement); + } + + found = testTokenVariations(lastElement); + } + return found; + } + + private Option testTokenVariations(String token) { + + Option opt; + if (nomLex.isPlural(token)) { + token = nomLex.getSingular(token); + } + + opt = testTokenInstance(token); + + if (!opt.isPresent()) { + if (token.endsWith("s")) { + String prefix = token.substring(0, token.length() - 1); + opt = testTokenInstance(prefix); + } + } + + if (!opt.isPresent()) { + + if (debug) { + System.out.println("Testing hard-coded suffixes"); + } + // hard coded lemma for bookmaker, steelmaker, downpayments, etc + if (token.endsWith("maker") || token.endsWith("makers")) + opt = new Option<>("maker"); + else if (token.endsWith("payment") || token.endsWith("payments")) + opt = new Option<>("payment"); + } + + if (!opt.isPresent()) { + if (debug) + System.out.println("Testing for counter+X"); + + if (token.startsWith("counter")) { + String suffix = token.replace("counter", ""); + opt = testTokenInstance(suffix); + } + } + + if (!opt.isPresent()) { + token = getPorterLemma(token); + opt = testTokenInstance(token); + } + + return opt; + } + + private Option testTokenInstance(String token) { + if (debug) { + System.out.println("Testing token " + token); + } + + if (nonStandard.containsKey(token)) { + token = nonStandard.get(token); + if (debug) { + System.out.println("Non standard lemma for token: " + token); + } + } + + Option found = Option.empty(); + if (nomLex.containsEntry(token)) { + + List entry = nomLex.getNomLexEntry(token); + + if (debug) { + System.out.println("Nomlex entry found: "); + for (NomLexEntry e : entry) { + System.out.println("\t" + e.nomClass); + } + } + + for (NomLexEntry e : entry) { + if (classes.contains(e.nomClass)) { + found = new Option<>(token); + break; + } + } + } + return found; + } + + private static synchronized String getPorterLemma(String token) { + stemmer.setCurrent(token); + stemmer.stem(); + return stemmer.getCurrent(); + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/nom/NomSRLManager.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/nom/NomSRLManager.java new file mode 100644 index 000000000..e2c23c85c --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/nom/NomSRLManager.java @@ -0,0 +1,140 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.nom; + +import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; +import edu.illinois.cs.cogcomp.srl.core.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; +import java.util.TreeSet; + +public class NomSRLManager extends SRLManager { + + private final static Logger log = LoggerFactory.getLogger(NomSRLManager.class); + + // XXX: Don't change these arrays unless you know what you are doing. If you + // change the order of elements, or adding or removing elements, then you + // have to re-train the Classifier. + private final static String[] coreArguments = {"A0", "A1", "A2", "A3", "A4", "A5", "A8", "A9"}; + + private final static String[] modifierArguments = {"AM-ADV", "AM-CAU", "AM-DIR", "AM-DIS", + "AM-EXT", "AM-LOC", "AM-MNR", "AM-NEG", "AM-PNC", "AM-PRD", "AM-TMP"}; + + private static final String[] allArguments = {NULL_LABEL, "A0", "A1", "A2", "A3", "A4", "A5", + "A8", "A9", "AM-ADV", "AM-CAU", "AM-DIR", "AM-DIS", "AM-EXT", "AM-LOC", "AM-MNR", + "AM-NEG", "AM-PNC", "AM-PRD", "AM-TMP", "C-A0", "C-A1", "C-A2", "C-A3", "C-SUP", + "R-A0", "R-A1", "R-A2", "R-A3", "R-A4", "R-A8", "R-AM-CAU", "R-AM-LOC", "SUP"}; + + private static final String[] allSenses = {"01", "02", "03", "04", "05", "06", "07", "08", + "09", "10", "11", "12", "13", "14"}; + + public final static Set coreArgumentSet = Collections.unmodifiableSet(new TreeSet<>( + Arrays.asList(coreArguments))); + + public static final Set modifierArgumentSet = Collections + .unmodifiableSet(new TreeSet<>(Arrays.asList(modifierArguments))); + + private ArgumentCandidateGenerator candidateGenerator; + private final AbstractPredicateDetector heuristicPredicateDetector; + + public NomSRLManager(boolean trainingMode, String defaultParser) throws Exception { + super(trainingMode, defaultParser); + candidateGenerator = new NomArgumentCandidateGenerator(this); + + this.heuristicPredicateDetector = new NomPredicateDetectorHeuristic(this); + } + + @Override + public SRLType getSRLType() { + return SRLType.Nom; + } + + @Override + protected String[] getArgumentLabels() { + return allArguments; + } + + @Override + protected String[] getSenseLabels() { + return allSenses; + } + + @Override + public Set getCoreArguments() { + return coreArgumentSet; + } + + @Override + public Set getModifierArguments() { + return modifierArgumentSet; + } + + @Override + public int getNumArguments() { + return allArguments.length; + } + + @Override + public int getNumSenses() { + return allSenses.length; + } + + @Override + public String getArgument(int id) { + return allArguments[id]; + } + + @Override + public String getSense(int id) { + return allSenses[id]; + } + + @Override + public ArgumentCandidateGenerator getArgumentCandidateGenerator() { + return this.candidateGenerator; + } + + @Override + public String getPredictedViewName() { + return ViewNames.SRL_NOM; + } + + public AbstractPredicateDetector getHeuristicPredicateDetector() { + return heuristicPredicateDetector; + } + + @Override + public AbstractPredicateDetector getLearnedPredicateDetector() { + try { + return new LearnedPredicateDetector(this); + } catch (Exception e) { + log.error("Unable to load the predicate detector!", e); + throw new RuntimeException(e); + } + } + + @Override + public int getPruneSize(Models model) { + switch (model) { + case Classifier: + return 7; + case Identifier: + return 6; + case Predicate: + case Sense: + return 4; + } + + return 4; + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/utilities/WeightVectorUtils.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/utilities/WeightVectorUtils.java new file mode 100644 index 000000000..1846c4d3d --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/utilities/WeightVectorUtils.java @@ -0,0 +1,135 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.utilities; + +import edu.illinois.cs.cogcomp.core.io.IOUtils; +import edu.illinois.cs.cogcomp.sl.util.WeightVector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.net.URL; +import java.util.List; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class WeightVectorUtils { + + private final static Logger log = LoggerFactory.getLogger(WeightVectorUtils.class); + + public static void save(String fileName, WeightVector wv) throws IOException { + BufferedOutputStream stream = + new BufferedOutputStream(new GZIPOutputStream(new FileOutputStream(fileName))); + + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(stream)); + + float[] w = wv.getWeightArray(); + + writer.write("WeightVector"); + writer.newLine(); + + writer.write(w.length + ""); + writer.newLine(); + + int numNonZero = 0; + for (int index = 0; index < w.length; index++) { + if (w[index] != 0) { + writer.write(index + ":" + w[index]); + writer.newLine(); + numNonZero++; + } + } + + writer.close(); + + log.info("Number of non zero weights: " + numNonZero); + } + + public static WeightVector load(String fileName) { + try { + GZIPInputStream zipin = new GZIPInputStream(new FileInputStream(fileName)); + + BufferedReader reader = new BufferedReader(new InputStreamReader(zipin)); + + String line; + + line = reader.readLine().trim(); + if (!line.equals("WeightVector")) { + reader.close(); + throw new IOException("Invalid model file."); + } + + line = reader.readLine().trim(); + int size = Integer.parseInt(line); + + WeightVector w = new WeightVector(size); + + while ((line = reader.readLine()) != null) { + line = line.trim(); + String[] parts = line.split(":"); + int index = Integer.parseInt(parts[0]); + float value = Float.parseFloat(parts[1]); + w.setElement(index, value); + } + + zipin.close(); + + return w; + } catch (Exception e) { + log.error("Error loading model file {}", fileName); + System.exit(-1); + } + return null; + } + + public static WeightVector loadWeightVectorFromClassPath(String fileName) { + try { + Class clazz = WeightVectorUtils.class; + List list = IOUtils.lsResources(clazz, fileName); + + if (list.size() == 0) { + log.error("File {} not found on the classpath", fileName); + throw new Exception("File not found on classpath"); + } + InputStream stream = list.get(0).openStream(); + + GZIPInputStream zipin = new GZIPInputStream(stream); + + BufferedReader reader = new BufferedReader(new InputStreamReader(zipin)); + + String line; + + line = reader.readLine().trim(); + if (!line.equals("WeightVector")) { + reader.close(); + throw new IOException("Invalid model file."); + } + + line = reader.readLine().trim(); + int size = Integer.parseInt(line); + + WeightVector w = new WeightVector(size); + + while ((line = reader.readLine()) != null) { + line = line.trim(); + String[] parts = line.split(":"); + int index = Integer.parseInt(parts[0]); + float value = Float.parseFloat(parts[1]); + w.setElement(index, value); + } + + zipin.close(); + return w; + } catch (Exception e) { + log.error("Error loading model file {}", fileName); + System.exit(-1); + } + return null; + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/verb/VerbPredicateDetector.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/verb/VerbPredicateDetector.java new file mode 100644 index 000000000..d5c9da3d0 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/verb/VerbPredicateDetector.java @@ -0,0 +1,126 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.verb; + +import edu.illinois.cs.cogcomp.core.datastructures.Option; +import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.SpanLabelView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TokenLabelView; +import edu.illinois.cs.cogcomp.edison.annotators.WordNetPlusLemmaViewGenerator; +import edu.illinois.cs.cogcomp.nlp.utilities.POSUtils; +import edu.illinois.cs.cogcomp.srl.core.AbstractPredicateDetector; + +public class VerbPredicateDetector extends AbstractPredicateDetector { + + public VerbPredicateDetector(VerbSRLManager manager) { + super(manager); + } + + @Override + public Option getLemma(TextAnnotation ta, int tokenId) { + String pos = POSUtils.getPOS(ta, tokenId); + String token = ta.getToken(tokenId).toLowerCase(); + TokenLabelView lemmaView = (TokenLabelView) ta.getView(ViewNames.LEMMA); + String lemma = lemmaView.getConstituentAtToken(tokenId).getLabel(); + boolean predicate = false; + + // any token that is a verb is a predicate + if (POSUtils.isPOSVerb(pos) && !pos.equals("AUX")) { + if (token.equals("'s") || token.equals("'re") || token.equals("'m")) + lemma = "be"; + else if (token.equals("'d") || lemma.equals("wo") || lemma.equals("'ll")) + lemma = "xmodal"; + + predicate = true; + + // modals and some + if (lemma.equals("xmodal") || pos.equals("MD") || token.equals("'ve")) + predicate = false; + + // ignore all instances of has + "to be" if they are followed by a + // verb or if the token is "be" followed by a verb + + boolean doVerb = lemma.equals("do"); + boolean be = lemma.equals("be"); + boolean have = lemma.equals("have"); + + if (tokenId < ta.size() - 1) { + + if (be) { + SpanLabelView chunk = (SpanLabelView) ta.getView(ViewNames.SHALLOW_PARSE); + for (Constituent c : chunk.getConstituentsCoveringToken(tokenId)) { + // if the token under consideration is not the last + // token, then there is another verb here + if (c.getEndSpan() - 1 != tokenId) { + predicate = false; + break; + } + } + } + + // ignore "have + be" + if (have && lemmaView.getConstituentAtToken(tokenId + 1).getLabel().equals("be")) { + predicate = false; + } + + // ignore "have/do + verb" + if ((have || doVerb) && POSUtils.isPOSVerb(POSUtils.getPOS(ta, tokenId + 1))) + predicate = false; + + // for some reason "according" in 'according to' is tagged as a + // verb. we want to avoid this. + + if (token.equals("according") + && ta.getToken(tokenId + 1).toLowerCase().equals("to")) + predicate = false; + } + + if (tokenId < ta.size() - 2) { + // ignore don't + V or haven't + V + if (doVerb || have) { + String nextToken = ta.getToken(tokenId + 1).toLowerCase(); + + if ((nextToken.equals("n't") || nextToken.equals("not")) + && POSUtils.isPOSVerb(POSUtils.getPOS(ta, tokenId + 2))) + predicate = false; + + } + } + + // NOTE: Not treating "have to" cases as exceptions because there + // are some instances annotated as verbs and some that aren't. + + // else if (has.contains(token)) { + // have to + + // except when the token is a variant of "has to" + // if (tokenId < ta.size() - 1 + // && ta.getToken(tokenId + 1).toLowerCase().equals("to")) + // predicate = false; + // } else { + // or when there is another VP following the token (Not sure why + // this would happen, though...) + // predicate &= checkVPAmongSiblings(ta, tokenId); + + // } + + } else if (token.startsWith("re-")) { + String trim = token.replace("re-", ""); + predicate = WordNetPlusLemmaViewGenerator.lemmaDict.contains(trim); + } + + if (predicate) { + + return new Option<>(lemma); + } else { + return Option.empty(); + } + } +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/verb/VerbSRLManager.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/verb/VerbSRLManager.java new file mode 100644 index 000000000..ec6474612 --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/verb/VerbSRLManager.java @@ -0,0 +1,154 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.verb; + +import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; +import edu.illinois.cs.cogcomp.srl.core.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; +import java.util.TreeSet; + +public class VerbSRLManager extends SRLManager { + private final static Logger log = LoggerFactory.getLogger(VerbSRLManager.class); + + // XXX: Don't change these arrays unless you know what you are doing. If you + // change the order of elements, or adding or removing elements, then you + // have to re-train the Classifier. + private static final String[] coreArguments = {"A0", "A1", "A2", "A3", "A4", "A5", "AA"}; + + private static final String[] modifierArguments = {"AM-ADV", "AM-CAU", "AM-DIR", "AM-DIS", + "AM-EXT", "AM-LOC", "AM-MNR", "AM-MOD", "AM-NEG", "AM-PNC", "AM-PRD", "AM-REC", + "AM-TMP"}; + + private static final String[] allArguments = {NULL_LABEL, "A0", "A1", "A2", "A3", "A4", "A5", + "AA", "AM-ADV", "AM-CAU", "AM-DIR", "AM-DIS", "AM-EXT", "AM-LOC", "AM-MNR", "AM-MOD", + "AM-NEG", "AM-PNC", "AM-PRD", "AM-REC", "AM-TMP", "C-A0", "C-A1", "C-A2", "C-A3", + "C-AM-ADV", "C-AM-CAU", "C-AM-DIS", "C-AM-EXT", "C-AM-LOC", "C-AM-MNR", "R-A0", "R-A1", + "R-A2", "R-A3", "R-AA", "R-AM-ADV", "R-AM-LOC", "R-AM-MNR", "R-AM-PNC", "R-AM-TMP", + "C-V", "C-A4", "C-AM-DIR", "C-AM-NEG", "C-AM-PNC", "C-AM-TMP", "R-A4", "R-AM-CAU", + "R-AM-EXT"}; + + // XXX: Ignoring the sense "XX". + private static final String[] allSenses = {"01", "02", "03", "04", "05", "06", "07", "08", + "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21"}; + + public final static Set coreArgumentSet = Collections.unmodifiableSet(new TreeSet<>( + Arrays.asList(coreArguments))); + + public static final Set modifierArgumentSet = Collections + .unmodifiableSet(new TreeSet<>(Arrays.asList(modifierArguments))); + + private ArgumentCandidateGenerator candidateGenerator; + + private final AbstractPredicateDetector heuristicPredicateDetector; + + public VerbSRLManager(boolean trainingMode, String defaultParser) throws Exception { + super(trainingMode, defaultParser); + candidateGenerator = new XuePalmerCandidateGenerator(this); + heuristicPredicateDetector = new VerbPredicateDetector(this); + + for (int i = 0; i < this.getNumLabels(Models.Classifier); i++) { + assert i == this.getArgumentId(this.getArgument(i)); + } + } + + public AbstractPredicateDetector getHeuristicPredicateDetector() { + return heuristicPredicateDetector; + } + + @Override + public SRLType getSRLType() { + return SRLType.Verb; + } + + @Override + protected String[] getArgumentLabels() { + return allArguments; + } + + @Override + protected String[] getSenseLabels() { + return allSenses; + } + + @Override + public Set getCoreArguments() { + return coreArgumentSet; + } + + @Override + public Set getModifierArguments() { + return modifierArgumentSet; + } + + @Override + public int getNumArguments() { + return allArguments.length; + } + + @Override + public int getNumSenses() { + return allSenses.length; + } + + @Override + public String getArgument(int id) { + return allArguments[id]; + } + + @Override + public String getSense(int id) { + return allSenses[id]; + } + + @Override + public ArgumentCandidateGenerator getArgumentCandidateGenerator() { + return this.candidateGenerator; + } + + @Override + public String getPredictedViewName() { + return ViewNames.SRL_VERB; + } + + @Override + public AbstractPredicateDetector getLearnedPredicateDetector() { + // for the verbs, we don't want a learned predicate detector because the + // POS based heuristic works very well. + // return getHeuristicPredicateDetector(); + if (this.getSRLType() == SRLType.Nom) { + try { + return new LearnedPredicateDetector(this); + } catch (Exception e) { + log.error("Unable to create learned predicate detector!", e); + throw new RuntimeException(e); + } + } else { + return getHeuristicPredicateDetector(); + } + } + + @Override + public int getPruneSize(Models model) { + switch (model) { + case Classifier: + return 6; + case Identifier: + return 4; + default: + return 4; + + } + } + +} diff --git a/srl/src/main/java/edu/illinois/cs/cogcomp/srl/verb/XuePalmerCandidateGenerator.java b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/verb/XuePalmerCandidateGenerator.java new file mode 100644 index 000000000..8b9ee5e4e --- /dev/null +++ b/srl/src/main/java/edu/illinois/cs/cogcomp/srl/verb/XuePalmerCandidateGenerator.java @@ -0,0 +1,140 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl.verb; + +import edu.illinois.cs.cogcomp.core.datastructures.IntPair; +import edu.illinois.cs.cogcomp.core.datastructures.Pair; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.Constituent; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.datastructures.trees.Tree; +import edu.illinois.cs.cogcomp.nlp.utilities.ParseTreeProperties; +import edu.illinois.cs.cogcomp.nlp.utilities.ParseUtils; +import edu.illinois.cs.cogcomp.srl.core.ArgumentCandidateGenerator; +import edu.illinois.cs.cogcomp.srl.core.SRLManager; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class XuePalmerCandidateGenerator extends ArgumentCandidateGenerator { + + public XuePalmerCandidateGenerator(SRLManager manager) { + super(manager); + } + + private final static Logger log = LoggerFactory.getLogger(XuePalmerCandidateGenerator.class); + + @Override + public String getCandidateViewName() { + return "XuePalmerHeuristicView"; + } + + @Override + public List generateCandidates(Constituent predicate) { + Constituent predicateClone = predicate.cloneForNewView(getCandidateViewName()); + + TextAnnotation ta = predicateClone.getTextAnnotation(); + int sentenceId = ta.getSentenceId(predicateClone); + Tree tree = ParseUtils.getParseTree(manager.defaultParser, ta, sentenceId); + + Tree> spanLabeledTree = ParseUtils.getSpanLabeledTree(tree); + + int sentenceStart = ta.getSentence(sentenceId).getStartSpan(); + int predicatePosition = predicateClone.getStartSpan() - sentenceStart; + + Set out = new HashSet<>(); + + List>> yield = spanLabeledTree.getYield(); + + if (predicatePosition >= yield.size()) { + System.out.println(ta); + + System.out.println("Predicate: " + predicatePosition + "\t" + predicateClone); + System.out.println(tree); + System.out.println(spanLabeledTree); + + System.out.println("Tree view"); + System.out.println(ta.getView(manager.defaultParser)); + + throw new RuntimeException(); + } + + Tree> predicateTree = yield.get(predicatePosition); + + Tree> currentNode = predicateTree.getParent(); + + boolean done = false; + while (!done) { + if (currentNode.isRoot()) + done = true; + else { + List candidates = new ArrayList<>(); + + for (Tree> sibling : currentNode.getParent().getChildren()) { + Pair siblingNode = sibling.getLabel(); + + // do not take the predicate as the argument + IntPair siblingSpan = siblingNode.getSecond(); + if (siblingSpan.equals(predicateClone.getSpan())) + continue; + + // do not take any constituent including the predicate as an argument + if ((predicatePosition >= siblingSpan.getFirst()) + && (predicateClone.getEndSpan() <= siblingSpan.getSecond())) + continue; + + String siblingLabel = siblingNode.getFirst(); + + int start = siblingSpan.getFirst() + sentenceStart; + int end = siblingSpan.getSecond() + sentenceStart; + + candidates.add(getNewConstituent(ta, predicateClone, start, end)); + + if (siblingLabel.startsWith("PP")) { + for (Tree> child : sibling.getChildren()) { + int candidateStart = + child.getLabel().getSecond().getFirst() + sentenceStart; + int candidateEnd = + child.getLabel().getSecond().getSecond() + sentenceStart; + + candidates.add(getNewConstituent(ta, predicateClone, candidateStart, + candidateEnd)); + } + } + } + out.addAll(candidates); + + currentNode = currentNode.getParent(); + } + } + + // Punctuations maketh an argument not! + List output = new ArrayList<>(); + for (Constituent c : out) { + if (!ParseTreeProperties.isPunctuationToken(c.getSurfaceForm())) + output.add(c); + } + + if (log.isDebugEnabled()) { + Exception ex = new Exception(); + String callerClass = ex.getStackTrace()[1].getClassName(); + String callerMethod = ex.getStackTrace()[1].getMethodName(); + int lineNumber = ex.getStackTrace()[1].getLineNumber(); + String caller = callerClass + "." + callerMethod + ":" + lineNumber; + + log.debug("Candidates for {} from heuristic: {}. Call from {}", new String[] { + predicateClone.toString(), output.toString(), caller}); + } + + return output; + } +} diff --git a/srl/src/main/resources/Nom.cross-predicate.modifiers b/srl/src/main/resources/Nom.cross-predicate.modifiers new file mode 100644 index 000000000..b2f8b8bc7 --- /dev/null +++ b/srl/src/main/resources/Nom.cross-predicate.modifiers @@ -0,0 +1,8 @@ +AM-DIR climb,A1 flight,A3 probe,A2 +AM-EXT %,A1 creditor,A1 interest,A2 maximum,A2 percentage,A1 rate,A2 scale,A2 +AM-LOC %,A1 account,A0 agent,A2 attendance,A1 bond,A2 brokerage,A2 buying,A0 center,A1 chief,A2 commuter,A1 corruption,A1 cost,A1 dealing,A0 departure,A1 designer,A2 director,A2 discrimination,A0 distance,A2 drawing,A2 earning,A3 equity,A1 exploration,A0 flight,A1 government,A2 guest,A1 home,A1 increase,A0 index,A0 industry,A1 information,A1 intelligence,A1 introduction,A2 investment,A2 jury,A2 land,A1 library,A2 life,A0 linage,A3 loan,A0 manager,A2 market,A2 mercy,A1 news,A1 officer,A2 official,A2 operating,A1 operation,A0 output,A0 page,A1 partner,A2 peace,A1 percentage,A1 pipeline,A0 plunge,A1 president,A2 pressure,A1 price,A1 principal,A1 problem,A1 profitability,A1 program,A0 rate,A1 record,A1 resident,A1 resource,A2 revenue,A2 room,A1 sale,A0 security,A1 share,A1 site,A2 slowdown,A1 specialist,A2 staff,A2 stake,A0 start-up,A1 supply,A2 system,A0 tariff,A3 termination,A0 tourist,A1 trade,A0 trading,A1 union,A3 volume,A3 +AM-MNR average,A2 bitterness,A2 buddy,A0 business,A1 charge,A1 fraud,A1 index,A1 law,A3 level,A2 measure,A1 problem,A2 productivity,A2 program,A1 project,A1 rate,A2 saving,A1 statute,A1 style,A2 system,A1 team,A1 trader,A1 unemployment,A2 volume,A2 +AM-NEG creativity,SUP +AM-PNC bill,A1 brokerage,A1 caution,A1 consent,A1 control,A1 development,A1 enforcement,A1 funding,A1 gain,A1 improvement,A1 investment,A2 sign,A1 thank,A2 trading,A4 +AM-PRD right,A1 +AM-TMP %,A1 cost,A1 era,A1 growth,A5 interest,A0 lifetime,A2 limit,A0 penalty,A3 season,A2 diff --git a/srl/src/main/resources/Nom.legal.arguments b/srl/src/main/resources/Nom.legal.arguments new file mode 100644 index 000000000..4e4dcdbd6 --- /dev/null +++ b/srl/src/main/resources/Nom.legal.arguments @@ -0,0 +1,4704 @@ +1-slash-10th 01#A1,A3 +abandonment 01#A1,A2,A0 +abatement 01#A1,A0 +abduction 01#A1,A0 +aberration 01#A0 +ability 01#A1,A0 +abolition 01#A1,A0 +abomination 01#A1,A0 +abortion 01#A1,A0 +absence 01#A1,A2 +absenteeism 01#A1,A2 +absorber 01#A1,A0 +absorption 01#A1,A0 +abstract 01#A1,A0 +absurdity 01#A1,A2 +abuse 01#A1,A0 02#A1,A0 +acceleration 01#A1,A2,A3,A4,A0 +accelerator 01#A1,A2,A3,A4,A0 +accent 01#A1,A2,A0 02#A1,A0 +acceptance 01#A1,A2,A3,A0 +access 01#A1,A0 +accessory 01#A1,A2,A0 +accident 01#A0 02#A1 +acclaim 01#A1,A2,A3,A0 +accolade 01#A1,A2,A0 +accommodation 01#A1,A0 +accompaniment 01#A1,A3,A4,A0 +accompanist 01#A1,A3,A4,A0 +accomplishment 01#A1,A0 +accord 01#A1,A2,A0 +accordance 01#A1,A2,A0 +account 01#A1,A2,A3,A0 02#A1,A0 03#A1,A2,A0 +accountability 01#A1,A3,A0 +accountant 01#A1,A2,A0 +accounting 01#A1,A3,A0 +accrual 01#A1,A0 +accumulation 01#A1,A3,A0 +accuracy 01#A1,A2 +accusation 01#A1,A2,A0 +accuser 01#A1,A2,A0 +achievement 01#A1,A0 +acknowledgement 01#A1,A2,A3,A0 +acquirer 01#A1,A2,A3,A4,A0 +acquisition 01#A1,A2,A3,A4,A0 +acquittal 01#A1,A2,A0 +acre 01#A1 +acrimony 01#A1,A2,A0 +acronym 01#A1,A2 +act 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A0 04#A1,A3 +acting 01#A1,A0 +action 01#A1,A2,A0 02#A1,A2,A3,A0 +activism 01#A1,A0 +activist 01#A1,A0 +activity 01#A1,A0 +actor 01#A1,A0 +actress 01#A1,A0 +ad 01#A1,A0 +adaptation 01#A1,A2,A3,A0 +adapting 01#A1,A2,A3,A0 +addict 01#A1,A2,A0 +addiction 01#A1,A2,A0 +addition 01#A1,A2 02#A1,A2,A0 +address 01#A1,A2,A0 02#A1,A2,A0 +adequacy 01#A1,A2 +adhesive 01#A1,A2,A0 +adjuster 01#A1,A2,A3,A4,A0 +adjustment 01#A1,A2,A3,A4,A0 +administration 01#A1,A2,A3,A0 +administrator 01#A1,A2,A3,A0 +admirer 01#A1,A0 +admission 01#A1,A2,A0 02#A1,A2,A0 +admonition 01#A1,A2,A0 +ado 01#A1,A0 +adoption 01#A1,A2,A0 +advance 01#A1,A2,A3,A4,A0 02#A1,A2,A0 +advancement 01#A1,A2,A3,A4,A0 +advancer 01#A1,A2,A3,A4,A0 +advantage 01#A1,A2,A0 +advent 01#A1,A0 +adventure 01#A1,A2 +adventurism 01#A0 +adversary 01#A1,A2,A0 +advertisement 01#A1,A0 +advertiser 01#A1,A0 +advertising 01#A1,A0 +advice 01#A1,A2,A0 +adviser 01#A1,A2,A0 +advisory 01#A1,A2,A0 +advocacy 01#A1,A2,A0 +advocate 01#A1,A2,A4,A0 +aegis 01#A1,A0 +aesthetic 01#A1,A2 +affair 01#A1,A0 02#A1,A0 +affection 01#A1,A0 +affidavit 01#A1,A2,A0 +affiliate 01#A1,A2,A0 +affiliation 01#A1,A2,A0 +affirmation 01#A1,A2,A0 +affliction 01#A1,A2,A0 +affordability 01#A1,A2,A0 +aftereffect 01#A1,A2,A0 +aftermath 01#A1,A2 +aftershock 01#A1,A2 +age 01#A1,A2 02#A1 +agenda 01#A1,A0 +agent 01#A1,A2,A0 02#A1,A0 +aggressiveness 01#A1,A0 +agility 01#A1,A2,A3,A0 +aging 01#A1,A2,A0 +agitation 01#A1,A0 +agreement 01#A1,A2,A0 +aid 01#A1,A2,A0 +aide 01#A1,A2,A0 +ailment 01#A1,A3,A0 +aim 01#A1,A0 02#A1,A2,A0 +air 01#A1,A2,A0 02#A1,A0 03#A1 04#A1,A0 +airing 01#A1,A2,A0 +alarm 01#A1,A0 02#A1,A2,A0 +album 01#A1,A0 +alchemist 01#A1,A0 +alcoholism 01#A1,A2 +alert 01#A1,A2,A0 +allegation 01#A1,A2,A3,A0 +allegiance 01#A1,A2,A0 +allergy 01#A1,A0 +alliance 01#A1,A2,A0 +allocation 01#A1,A2,A0 +allocator 01#A1,A2,A0 +allotment 01#A1,A2,A0 +allowance 01#A1,A2,A0 +allure 01#A1,A2,A3 +allusion 01#A1,A0 +ally 01#A1,A2,A3,A0 +alternative 01#A1,A2,A3,A0 +altitude 01#A1,A2 +alumnus 01#A1,A0 +amalgamation 01#A1,A0 +amazement 01#A1,A0 +ambassador 01#A1,A2,A0 +ambiguity 01#A1,A2 +ambition 01#A1,A0 +ambivalence 01#A1,A0 +amendment 01#A1,A2,A0 +amenity 01#A1,A0 +amnesty 01#A1,A0 +amount 01#A1,A2,A3 +analysis 01#A1,A0 +analyst 01#A1,A2,A0 +anathema 01#A0 +ancestry 01#A1,A2 +anchor 01#A1,A0 +anger 01#A1,A0 +angle 01#A1,A2,A0 +angst 01#A1,A2,A0 +anguish 01#A1,A0 +annals 01#A1,A0 +annex 01#A1 +anniversary 01#A1,A2,A0 +announcement 01#A1,A2,A3,A0 +announcer 01#A1,A2,A3,A0 +anonymity 01#A1,A2 +answer 01#A1,A2,A3,A0 +antagonist 01#A1,A0 +ante 01#A1,A2,A0 +antecedent 01#A1,A0 +anthem 01#A1,A3,A0 +anther 01#A1 +anthology 01#A1,A2,A3,A0 +antic 01#A1,A0 +anticipation 01#A1,A0 +antidote 01#A1,A2,A3,A0 +anxiety 01#A1,A0 +apartment 01#A1 +apologist 01#A1,A2,A0 +apology 01#A1,A2,A0 +apparat 01#A1,A0 +apparatus 01#A1,A0 +appeal 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 +appearance 01#A1 02#A1,A2 +appetite 01#A1,A2 02#A1,A0 +applause 01#A1,A2,A0 +applicability 01#A1,A2,A0 +applicant 01#A1,A2,A0 +application 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A3,A0 +appointee 01#A1,A2,A0 +appointment 01#A1,A2,A0 02#A1,A2,A0 +appraisal 01#A1,A2,A0 +appraiser 01#A1,A2,A0 +appreciation 01#A1,A2,A3,A4 02#A1,A0 +apprehension 01#A1,A0 +approach 01#A1,A2,A0 +appropriateness 01#A1,A2 +appropriation 01#A1,A2,A3,A0 +approval 01#A1,A2,A3,A0 +arbitrage 01#A1,A2,A3,A4,A0 +arbitrager 01#A1,A2,A3,A4,A0 +arbitration 01#A1,A2,A0 +architect 01#A1,A2,A0 +architecture 01#A1,A2,A0 +archness 01#A1,A2 +archrival 01#A1,A0 +ardor 01#A1,A0 +area 01#A1,A2 02#A1,A2,A3 03#A1 04#A1,A0 05#A1,A3 +arena 01#A1,A2,A3 +argument 01#A1,A2,A0 02#A1,A2,A0 +arithmetic 01#A1 +arm 01#A1,A2 02#A1 +armpit 01#A1 +army 01#A1,A2,A3 +aroma 01#A1 02#A1 +array 01#A1,A3 +arrears 01#A1,A2,A3,A0 +arrest 01#A1,A2,A0 +arrival 01#A1,A2,A3,A4 +arrogance 01#A1,A0 +arsenal 01#A1,A0 +art 01#A1,A0 02#A1,A2 +artery 01#A2,A3,A4 02#A1,A3 +article 01#A1,A2,A0 +artifact 01#A1,A2 +artist 01#A1,A0 +artistry 01#A0 +artwork 01#A1,A0 +ascent 01#A1,A0 +asking 01#A1,A2,A0 02#A1,A2,A3,A0 +aspect 01#A1,A2 +aspersion 01#A1,A0 +ass 01#A1,A2,A3,A0 +assailant 01#A1,A2,A0 +assassination 01#A1,A0 +assault 01#A1,A2,A0 +assembly 01#A1,A2,A3 02#A1,A2,A4,A0 +assent 01#A1,A2,A0 +assertion 01#A1,A0 +assessment 01#A1,A2,A0 +assignment 01#A1,A2,A0 +assistance 01#A1,A2,A0 +assistant 01#A1,A2,A3,A0 02#A1 +associate 01#A1,A2,A0 +association 01#A1,A2,A3 02#A1,A2,A0 +assortment 01#A1,A3 +assumption 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +assurance 01#A1,A2,A0 +astonishment 01#A1,A2,A0 +astringency 01#A1,A2 +asylum 01#A1,A2,A3,A0 +atmosphere 01#A1,A2,A3 02#A1 +atom 01#A1,A3 +atrocity 01#A1,A2,A0 +attachment 01#A1,A2,A3,A0 +attack 01#A1,A2,A0 02#A1,A2,A0 +attacker 01#A1,A2,A0 +attempt 01#A1,A0 +attendance 01#A1,A0 +attendant 01#A1,A0 +attendee 01#A1,A0 +attention 01#A1,A0 +attitude 01#A1,A2,A0 +attorney 01#A1,A2,A3,A4,A0 +attraction 01#A1,A2,A3,A0 +attractiveness 01#A1,A0 +attribute 01#A1,A2 +auction 01#A1,A2,A3,A4,A0 +auctioneer 01#A1,A2,A3,A4,A0 +audacity 01#A1,A0 +audience 01#A1,A0 +audit 01#A1,A0 +audition 01#A1,A2,A0 +auditor 01#A1,A0 +aunt 01#A1,A0 +aura 01#A1 +authenticity 01#A1,A2,A3 +author 01#A1,A0 +authority 01#A1,A2,A3,A0 +authorization 01#A1,A2,A3,A0 +authorship 01#A1,A2 +autobiography 01#A1,A0 +autograph 01#A1,A0 +automation 01#A1,A0 +autonomy 01#A1,A2 +autumn 01#A1 +availability 01#A1,A2,A3 +avalanche 01#A1,A3 +avenue 01#A1 02#A1 +average 01#A1,A2,A3,A0 +averaging 01#A1,A2,A3,A0 +avoidance 01#A1,A0 +award 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A0 +awarding 01#A1,A2,A0 +awareness 01#A1,A0 +axiom 01#A1,A2,A3,A0 +babel 01#A1,A3 +baby 01#A1,A0 +back 01#A1 02#A1,A2,A3,A0 +back-up 01#A1,A0 02#A1,A2,A0 03#A1 +backbone 01#A1,A0 02#A1 +backdrop 01#A1,A2,A3 +backer 01#A1,A0 +background 01#A1,A2,A3 +backing 01#A1,A0 +backlash 01#A1,A0 +backlog 01#A1,A2,A3,A0 +backside 01#A1 +badge 01#A1,A0 +bag 01#A1 +bagger 01#A1,A0 +bail 01#A1,A2,A3,A0 +bailout 01#A1,A2,A3,A0 +baker 01#A1,A2,A3,A0 +balance 01#A1,A2,A0 02#A1,A2,A3 +balloting 01#A1,A0 +ban 01#A1,A2,A0 +band 01#A1,A2,A3 02#A1,A2,A3 +bane 01#A1,A2,A0 +banker 01#A1,A2,A0 +bankruptcy 01#A1 +banner 01#A1,A2,A0 +banning 01#A1,A2,A0 +banter 01#A1,A2,A0 +bargain 01#A1,A2,A0 +bargaining 01#A1,A2,A0 +barometer 01#A1,A2,A3,A0 +baron 01#A1,A2,A0 +barrage 01#A1,A2,A0 +barrel 01#A1,A3 02#A1 +barrier 01#A1,A2,A3,A0 +bartering 01#A1,A2,A3,A0 +base 01#A1 02#A1 03#A1,A2,A3 04#A1,A3 05#A1,A2,A0 06#A1,A0 07#A1 +baseline 01#A1,A2 +basement 01#A1 +bashing 01#A1,A0 +basis 01#A1,A2,A0 02#A1,A0 +basket 01#A1,A3 +bassist 02#A1,A2,A0 +bastion 01#A1,A2,A3,A0 +batch 01#A1,A3 +bathroom 01#A1 +battalion 01#A1,A2,A3 +battery 01#A1,A3 +batting 01#A1,A0 02#A1,A0 +battle 01#A1,A2,A0 +battlement 01#A1 02#A1,A2,A3,A0 +bearer 01#A1,A2,A3,A0 +bearing 01#A1,A2 02#A1,A0 03#A1,A0 04#A1,A2,A0 +beauty 01#A1,A2,A3 +bedrock 01#A1,A0 +beginning 01#A1,A2,A0 +behalf 01#A1,A2,A0 +behavior 01#A1,A0 +beheading 01#A1,A0 +behemoth 01#A1,A3,A0 +behest 01#A1,A2,A0 +being 01#A1,A2 +belief 01#A1,A0 +believer 01#A1,A0 +bellwether 01#A1,A0 +belly 01#A1 +benchmark 01#A1,A2,A3 +benefactor 01#A1,A0 +beneficiary 01#A1,A0 +benefit 01#A1,A0 +bent 01#A1,A2 +bequest 01#A1,A2,A0 +bet 01#A1,A2,A3,A0 +beta 01#A1,A2 +better 01#A1,A0 +betterment 01#A1,A0 +bevy 01#A1,A3 +bias 01#A1,A0 +bible 01#A1,A0 +bicentennial 01#A1,A0 +bickering 01#A1,A0 +bid 01#A1,A2,A0 +bidder 01#A1,A2,A0 +bidding 01#A1,A2,A0 02#A1,A0 +bill 01#A1,A2,A3,A0 02#A1,A3,A0 03#A1,A2,A0 04#A1,A2 05#A1 +billing 01#A1,A2,A3,A0 02#A1,A2 +billions 01#A1,A3 +binge 01#A1,A0 +biographer 01#A1,A2,A0 +biography 01#A1,A0 +birth 01#A1 +birthday 01#A1,A2 +birthplace 01#A1,A3,A0 +bishop 01#A1,A2,A0 +bit 01#A1,A3 02#A1 +bite 01#A1,A2,A3 02#A1,A0 +bitterness 01#A1,A2,A3,A0 +blacklisting 01#A1,A2,A0 +blame 01#A1,A2,A0 +blanket 01#A1,A3 02#A1 +blast 01#A1,A0 02#A1,A2,A0 +bleeding 01#A1,A0 +blemish 01#A1,A0 +blend 01#A1,A2,A3,A0 +blessing 01#A1,A2,A0 +blink 01#A1,A0 02#A1 +blip 01#A1,A2,A3,A4,AM +blitz 01#A1,A3 +bloc 01#A1,A2,A3 +block 01#A1 +blockade 01#A1,A2,A3,A0 +blocker 01#A1,A2,A3,A0 +blow 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +blueprint 01#A1,A0 +blunder 01#A1,A0 +board 01#A1,A2,A3 +boardroom 01#A1 +boast 01#A1,A2,A0 +boatload 01#A1,A3 +body 01#A1 02#A1,A3 03#A1,A2,A3 +boilerplate 01#A1 +bombardment 01#A1,A2,A0 +bombing 01#A1,A0 +bond 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +bone 01#A1 02#A1,A0 +bonus 01#A1,A2,A0 +book 01#A1,A0 02#A1 03#A1,A2,A3,A0 +booking 01#A1,A2,A0 02#A1,A2,A3,A0 +booklet 01#A1,A0 +boom 01#A0 +boon 01#A1,A2 +boost 01#A1,A2,A3,A4,AM,A0 +booster 01#A1,A2,A3,A4,AM,A0 +border 01#A1,A2 +boredom 01#A1,A2,A0 +borough 01#A1 +borrower 01#A1,A2,A3,A0 +borrowing 01#A1,A2,A3,A0 +bottle 01#A1,A3 +bottleneck 01#A1,A2 +bottler 01#A1,A0 +bottling 01#A1,A0 +bottom 01#A1 +boundary 01#A1 +bounty 01#A1,A2,A0 +bout 01#A1,A3 +bowl 01#A1,A3 +box 01#A1,A3 +boy 01#A1,A0 02#A1,A2,A0 +bracket 01#A1,A2,A0 +brain 01#A1 +brainchild 01#A1,A0 +branch 01#A1,A2 02#A1 +brand 01#A1,A2,A0 +brass 01#A1,A2,A0 +brat 01#A1,A0 +brawl 01#A1,A2,A0 +breach 01#A1,A0 +breadbasket 01#A1,A3,A0 +breadth 01#A1,A2 +break 01#A1,A2,A3,A0 02#A0 03#A1,A2,A3,A4,A0 +break-up 01#A1,A2,A0 +breakage 01#A1,A2,A3,A0 +breakdown 01#A1,A0 02#A1,A0 +breaker 01#A1,A2,A3,A0 +breaking 01#A1,A2,A3,A0 +breakthrough 01#A1,A0 +breast 01#A1 +breath 01#A1,A2,A0 +breather 01#A0 +breathing 01#A1,A2,A0 +breed 01#A1 +breeder 01#A1,A0 +breeding 01#A1,A0 +brethren 01#A1,A2,A0 +brewing 01#A1,A2,A3,A0 +bribe 01#A1,A2,A3,A0 +bribery 01#A1,A2,A3,A0 +bride 01#A1,A0 +bridge 01#A1,A2,A0 +brief 01#A1,A2,A0 +briefing 01#A1,A2,A0 +brigade 01#A1,A2,A3 +bringing 01#A1,A2,A3,A4,A0 +brink 01#A1,A2,A3,A0 +broadcast 01#A1,A2,A3,A0 +broadcaster 01#A1,A2,A3,A0 +broadcasting 01#A1,A2,A3,A0 +broadside 01#A1,A0 +brochure 01#A1,A0 +broker 01#A1,A2,A0 +brokerage 01#A1,A2,A0 +brother 01#A1,A3,A0 +brouhaha 01#A1,A0 +brow 01#A1 +brunt 01#A1,A2,A3,A0 +brush 01#A1,A0 02#A1,A0 +brutality 01#A1,A0 +buck 01#A1,A2,A0 +bucket 01#A1,A3 +buckle 01#A1 +buddy 01#A1,A2,A0 +budget 01#A1,A2,A0 +buff 01#A1,A2,A0 +buffer 01#A1,A0 +buffet 01#A1,A3 +bugaboo 01#A1,A2,A3,A0 +build-up 01#A1,A2,A3,A4,A0 +builder 01#A1,A2,A4,A0 +building 01#A1,A2,A4,A0 +bulk 01#A1 02#A1 03#A1,A2,A3 +bulkhead 01#A1 +bulletin 01#A1,A0 +bullishness 01#A1,A0 +bulwark 01#A1,A2,A3,A0 +bump 01#A1 +bunch 01#A1,A3 +bundle 01#A1,A3 +bunt 01#A1,A2,A0 +buoyancy 01#A1,A2 +burden 01#A1,A2,A0 +bureau 01#A1,A2 +bureaucracy 01#A1,A2,A3 +bureaucrat 01#A1,A2,A0 +burglary 01#A1,A0 +burn 01#A1,A0 +burner 01#A1,A0 +burning 01#A1,A0 +burst 01#A1,A2 02#A1,A0 +bus 01#A2,A3,A4 +bushel 01#A1,A3 +business 01#A1,A0 02#A1,A0 03#A1,A2,A0 +busload 01#A1,A3 +buster 01#A1,A0 +bustle 01#A1,A2,A0 +butler 01#A1,A2,A0 +butt 01#A1,A2,A3,A4,A0 02#A1 +buy 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A4,A0 +buy-back 01#A1,A2,A3,A4,A0 +buy-out 01#A1,A2,A3,A4,A0 +buyer 01#A1,A2,A3,A4,A0 +buying 01#A1,A2,A3,A4,A0 +buzzsaw 01#A1,A3 +byline 01#A1,A2,A3 +cabinet 01#A1,A3 02#A1,A2,A3 +cache 01#A1,A3 +cachet 01#A1,A2,A3 +cafeteria 01#A1 +cage 01#A1,A2,A3,A0 +calculation 01#A1,A2,A0 +calendar 01#A1,A2,A3,A0 +caliber 01#A1,A2 +call 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 +caller 01#A1,A2,A0 +calling 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +calm 01#A1,A2 +calmness 01#A1,A2 +camaraderie 01#A1,A2,A0 +cameo 01#A1,A2,A0 +camp 01#A1,A2,A3 +campaign 01#A1,A0 02#A1,A0 +campaigner 01#A1,A0 +campus 01#A1,A2 +can 01#A1,A3 +cancellation 01#A1,A0 +cancer 01#A1,A2,A0 +candidacy 01#A1,A2,A0 +candidate 01#A1,A2,A3,A0 +candor 01#A1,A0 +canning 01#A1,A0 +cant 01#A1,A2,A0 +cap 01#A1,A2,A0 +capability 01#A1,A0 +capacity 01#A1,A0 +capital 01#A1,A0 02#A1,A2,A0 +capitalist 01#A1,A0 +capitalization 01#A1,A2,A0 +capsule 01#A1,A3 +captive 01#A1,A0 02#A1 +care 01#A1,A2,A0 +career 01#A1,A0 +cargo 01#A1,A3 +carnage 01#A1,A2,A0 +carping 01#A1,A0 +carrier 01#A1,A2,A3,A4,A0 +carry-forward 01#A1,A2,A3,A0 +carrying 01#A1,A2,A3,A4,A0 +cartel 01#A1,A2,A3 +carton 01#A1,A3 +cartoon 01#A1,A0 +cascade 01#A1,A3 +case 01#A1,A2,A3,A4,A0 02#A1,A3 03#A1,A2 04#A1,A2,A3,A0 05#A1,A2 06#A1 07#A1 +caseload 01#A1,A2,A0 +cast 01#A1 02#A1,A2,A0 +caster 01#A1,A2,A0 +casting 01#A1,A2,A0 +casualty 01#A1,A2,A3,A0 +casuistry 01#A1,A0 +cataclysm +catalyst 01#A1,A0 +catch 01#A1,A2,A0 02#A1,A2,A3 +category 01#A1,A2 +caterer 01#A1,A2,A0 +cause 01#A1,A0 02#A1,A2,A0 03#A1,A2,A3,A0 +caution 01#A1,A2,A0 +cautiousness 01#A1,A0 +cave-in 01#A1 +caveat 01#A1,A0 +cease-fire 01#A1,A2,A0 +ceiling 01#A1,A2 02#A1 +celebration 01#A1,A2,A0 02#A1,A0 +censorship 01#A1,A0 +census 01#A1,A0 +cent 01#A1 02#A1 +centennial 01#A1 +center 01#A1,A2,A3 +centerfielder 01#A1,A2,A0 +centerpiece 01#A1,A0 02#A1 +century 01#A1,A2,A3 02#A1,A2,A3 +ceremony 01#A1,A2,A0 +certainty 01#A1,A2,A0 +certificate 01#A1,A2,A0 +cessation 01#A1,A2,A0 +chain 01#A1,A2,A3 +chairman 01#A1,A2,A3,A0 +chairmanship 01#A1,A2,A0 +challenge 01#A1,A2,A0 +challenger 01#A1,A2,A0 +chamber 01#A1,A2 +champ 01#A1,A0 +champion 01#A1,A0 02#A1,A0 +championship 01#A1,A2,A0 02#A1,A2,A3,A0 +chance 01#A1,A2,A0 +chancellor 01#A1,A2,A0 +change 01#A1,A2,A3,A0 +changeover 01#A1,A2,A3,A0 +changing 01#A1,A2,A3,A0 +chant 01#A1,A0 +chaos 01#A1 +chapter 01#A1 02#A1,A2 +character 01#A1,A2 02#A1,A0 +characteristic 01#A1,A2 +characterization 01#A1,A2,A0 +charge 01#A1,A2,A3,A0 02#A1,A0 03#A1,A2,A0 04#A1,A2,A0 +charge-off 01#A1,A0 +charging 01#A1,A2,A0 +charisma 01#A1,A2,A3 +charlatanry 01#A1,A2,A0 +charm 01#A1,A2,A3 +chart 01#A1,A0 +charter 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +chase 01#A1,A0 +chaser 01#A1,A0 +chassis 01#A1 +chatter 01#A1,A2,A0 +chauvinism 01#A0 +cheater 01#A1,A0 +cheating 01#A1,A0 +check 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +checkpoint 01#A1,A0 +cheek 01#A1 +cheer 01#A1,A2,A0 +chef 01#A1,A2,A0 +chemist 01#A1,A2,A0 +chemistry 01#A1 +chest 01#A1,A3 +chief 01#A1 02#A1,A2,A3,A0 +child 01#A1,A0 +childhood 01#A1,A2,A3 02#A1,A0 +chin 01#A1 +chip 01#A1 +chit 01#A1,A0 +choice 01#A1,A2,A3,A4,A0 +choosing 01#A1,A2,A3,A4,A0 +chore 01#A1,A2,A3,A4,A0 +chorus 01#A1,A3 02#A1,A2,A3 +chronicle 01#A1,A0 +chum 01#A1,A2,A0 +chunk 01#A1,A2,A0 +church 01#A1,A3,A0 +chutzpah 01#A1,A0 +circle 01#A1,A2,A3 +circuit 01#A1 02#A0 +circulation 01#A1,A2 02#A1,A0 +circumstance 01#A1,A2 +citation 01#A1,A2,A3,A0 02#A1,A2,A0 +citizen 01#A1,A3,A0 +citizenry 01#A1,A3,A0 +citizenship 01#A1,A2,A0 +city 01#A1 +claim 01#A1,A0 02#A1,A2,A0 03#A1,A2,A3,A0 +claimant 01#A1,A2,A3,A0 +clamp 01#A1,A0 +clampdown 01#A1,A2,A3,A0 +clan 01#A1,A2,A3 +clanging 01#A1,A0 +clarification 01#A1,A2,A0 +clarity 01#A1,A0 +clash 01#A1,A2,A0 +class 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A2 +classification 01#A1,A2,A0 +classmate 01#A1,A2,A0 +claudication 01#A0 +clause 01#A1,A2,A0 +cleaner 01#A1,A2,A0 02#A1,A0 +cleanser 01#A1,A2,A0 +cleanup 01#A1,A0 +clearance 01#A1,A2,A3,A0 +clearing 01#A1,A0 +client 01#A1,A2,A3,A0 +clientele 01#A1,A2,A0 +climate 01#A1,A2,A3 +climb 01#A1,A0 02#A1,A2,A3,A4,AM +climber 01#A1,A0 +clip 01#A1,A2,A3 02#A1,A0 +clipping 01#A1,A2,A0 +cloak 01#A1,A2,A0 +clogging 02#A1,A2,A0 +clone 01#A1,A0 +close 01#A1,A2 +close-up 01#A1,A0 +closeness 01#A1,A2,A3 +closet 01#A1 02#A1 +closing 01#A1,A2,A0 02#A1,A2 03#A1,A0 +closure 01#A1,A2,A0 +cloture 01#A1,A2,A0 +cloud 01#A1,A3 +clouding 01#A1,A2,A0 +clout 01#A1,A2,A3,A0 +club 01#A1,A2,A3 +clue 01#A1,A2,A0 +clump 01#A1,A3 +cluster 01#A1,A3 +clutch 01#A1,A3 +clutter 01#A1 +coalition 01#A1,A2,A3 +coast 01#A1,A2 +coastline 01#A1,A2 +coat 01#A1,A2,A0 +coating 01#A1,A2,A0 +coattail 01#A1,A0 +cockpit 01#A1 +code 01#A1,A2,A3,A0 +coercion 01#A1,A2,A0 +cogeneration 01#A1,A2,A3,A4,A0 +cognoscenti 01#A1,A2,A0 +cohort 01#A1,A2,A0 +coincidence 01#A1 +cold 01#A1,A2 +collage 01#A1,A2,A3,A0 +collapse 01#A1,A0 02#A1,A2,A3,A4 +collar 01#A1,A2,A0 +collateral 01#A1,A2 +colleague 01#A1,A0 02#A1,A2,A0 +collectability 01#A1,A2,A4,A0 +collecting 01#A1,A2,A4,A0 +collection 01#A1,A2,A4,A0 +collector 01#A1,A2,A4,A0 +colloquium 01#A1,A2,AM,A0 +colonel 01#A1,A3,A0 +colony 01#A1,A0 +color 01#A1,A2 +colorization 01#A1,A2,A0 +colorlessness 01#A1 +column 01#A1,A2,A0 02#A1,A3 03#A1 +columnist 01#A1,A2,A0 +combat 01#A1,A2,A0 +combination 01#A1,A2,A3,A0 +combustion 01#A1,A2,A0 +comeback 01#A1,A2 +comedy 01#A1,A0 +comeuppance 01#A1,A2,A3,A0 +comfort 01#A1,A2,A0 +comity 01#A1 +command 01#A1,A0 02#A1,A2,A0 +commandant 01#A1,A2,A0 +commander 01#A1,A2,A0 +commando 01#A1,A0 +commencement 01#A1,A2,A0 +comment 01#A1,A2,A0 +commentary 01#A1,A2,A0 +commentator 01#A1,A2,A3,A0 +commerce 01#A1,A2,A3,A4,A0 +commercial 01#A1,A0 +commercialization 01#A1,A2,A0 +commission 01#A1,A0 02#A1,A2,A0 03#A1,A2,A3,A0 +commissioner 01#A1,A2,A3,A0 +commitment 01#A1,A2,A0 02#A1,A0 +committee 01#A1,A2,A3 +common 01#A1,A2,A0 +commonality 01#A1,A2,A0 +commonplace 01#A1 +commotion 01#A1,A2,A0 +communication 01#A1,A2,A0 +communique 01#A1,A2,A0 +community 01#A1,A2,A3 +commute 01#A1,A2,A0 +commuter 01#A1,A2,A0 +companion 01#A1,A0 02#A1,A2,A0 +comparability 01#A1,A2,A0 +comparison 01#A1,A2,A0 +compassion 01#A1,A0 +compatibility 01#A1,A2,A0 +compatriot 01#A1,A0 +compensation 01#A1,A2,A3,A0 +competence 01#A1,A0 +competition 01#A1,A2,A0 +competitiveness 01#A1,A0 +competitor 01#A1,A2,A0 +compilation 01#A1,A4,A0 +compiler 01#A1,A4,A0 +complacency 01#A1,A2,A0 +complaint 01#A1,A2,A3,A0 02#A1,A2,A0 +completion 01#A1,A0 +complex 01#A1,A2,A3 +complexity 01#A1,A2 +compliance 01#A1,A0 +complication 01#A1,A2,A0 +complicity 01#A1,A0 +compliment 01#A1,A2,A0 +component 01#A1 +composer 01#A1,A2,A0 +composition 01#A1,A2,A0 02#A1,A2,A0 +comprehension 01#A1,A0 +comprehensiveness 01#A1,A2 +compromise 01#A1,A2,A0 +comptroller 01#A1,A2,A0 +compulsion 01#A1,A0 +computer 01#A1,A2,A0 +computing 01#A1,A2,A0 +comrade 01#A1,A2,A0 +conceit 01#A1,A2,A0 +conceiver 01#A1,A2,A0 +concentrate 01#A1,A0 +concentration 01#A1,A2,A0 02#A1,A0 +concept 01#A1,A2,A0 +conception 01#A1,A2,A0 02#A0 +concern 01#A1,A2,A0 02#A1,A0 +concert 01#A1,A0 +concession 01#A1,A2,A0 +conclusion 01#A1,A0 02#A1,A2,AM,A0 +concoction 01#A1,A2,A3,A4,A0 +concomitant 01#A1,A2 +concurrence 01#A1,A0 +condemnation 01#A1,A2,A3,A0 +condescension 01#A2,A0 +condition 01#A1,A2,A3 02#A1,A2,A0 +conditioner 01#A1,A2,A0 +conduct 01#A1,A0 02#A1,A0 +conduit 01#A1,A2,A3 +conferee 01#A1,A0 +conference 01#A1,A2,A0 +confession 01#A1,A2,A0 +confidant 01#A1,A2,A0 +confidence 01#A1,A2,A0 +confidentiality 01#A1,A2,A0 +configuration 01#A1,A2,A3,A0 +confine 01#A1,A2,A0 +confinement 01#A1,A2,A0 +confirmation 01#A1,A2,A0 02#A1,A2,A0 +conflation 01#A1,A2,A3,A0 +conflict 01#A1,A2,A0 +confluence 01#A1,A2,A3,A0 +confrontation 01#A1,A2,A0 +confusion 01#A1,A2,A0 +conglomerate 01#A1,A2,A3 +congress 01#A1,A2,AM,A0 +congressman 01#A1,A0 +conjunction 01#A1,A2,A0 +connection 01#A1,A2,A0 +connoisseur 01#A1,A2,A0 +conquest 01#A1,A0 +conscience 01#A1,A0 +consciousness 01#A1,A0 +consensus 01#A1,A2,A0 +consent 01#A1,A2,A0 +consequence 01#A1,A2,A0 +conservation 01#A1,A0 +conservatism 01#A1,A0 +conservative 01#A1,A0 +conservatorship 01#A1,A0 +consideration 01#A1,A2,A0 02#A1,A0 03#A1 04#A1,A2,A3,A0 +consolation 01#A1,A2,A3,A0 +consolidation 01#A1,A2,A3,A0 +consortium 01#A1,A2,A3 +conspiracy 01#A1,A2,A0 +consternation 01#A1,A0 +constituency 01#A1,A0 +constituent 01#A1,A0 +constitutionality 01#A1,A2,A3 +constraint 01#A1,A2,A3,A0 +construction 01#A1,A2,A4,A0 02#A1,A0 +consultant 01#A1,A2,A0 +consultation 01#A1,A2,A0 +consulting 01#A1,A2,A0 +consumer 01#A1,A0 +consuming 01#A1,A0 +consumption 01#A1,A0 +contact 01#A1,A0 +container 01#A1,A0 +containment 01#A1,A0 +contamination 01#A1,A2,A0 +contemporary 01#A1,A0 +contempt 01#A1,A0 +contender 01#A1,A2,A0 +content 01#A1,A0 +contention 01#A1,A2,A0 02#A1,A2,A0 +contest 01#A1,A2,A0 02#A1,A0 +contestant 01#A1,A0 +context 01#A1 +contingent 01#A1,A2,A3 +continuation 01#A1,A0 +contract 01#A1,A2,A3,A0 +contracting 01#A1,A2,A3,A0 +contraction 01#A1,A2,A3,A4 02#A1,A2,A3,A4,AM +contractor 01#A1,A2,A3,A0 +contradiction 01#A1,A0 +contrary 01#A1,A2 +contrast 01#A1,A2,A0 +contribution 01#A1,A2,A0 +contributor 01#A1,A2,A0 +control 01#A1,A2,A0 +controller 01#A1,A2,A0 +controlling 01#A1,A2,A0 +controversy 01#A1,A0 +convenience 01#A1,A2 +convention 01#A1,A0 +conversation 01#A1,A2,A3,A0 +conversion 01#A1,A2,A3,A0 +converter 01#A1,A2,A3,A0 +convertibility 01#A1,A2,A3,A0 +convexity 01#A1,A2,A3 +conveyance 01#A1,A2,A3,A4,A0 +conviction 01#A1,A2,A0 02#A1,A0 +convocation 01#A1,A2,AM,A0 +convoy 01#A1,A2,A3 +convulsion 01#A1,A0 +cooking 01#A1,A0 +cool 01#A1,A2 +cooling 01#A1,A0 +cooperation 01#A1,A2,A0 +coordinator 01#A1,A2,A3,A0 +copy 01#A1,A2,A0 +copying 01#A1,A2,A0 +copyright 01#A1,A0 +cord 01#A1 +core 01#A1,A3,A0 02#A1 03#A1,A3 +corner 01#A1 02#A1,A2 03#A1,A0 +cornerstone 01#A1,A0 +cornucopia 01#A1,A3 +corporatism 01#A1,A2 +corps 01#A1,A2,A3 +correction 01#A1,A0 +correspondence 01#A1,A2,A0 02#A1,A2 +correspondent 01#A1,A2,A3,A0 +corruption 01#A1,A2,A3,A0 +cost 01#A1,A2,A3,A4 +council 01#A1,A2,A3 +councilman 01#A1,A0 +councilwoman 01#A1,A2,A3,A0 +counsel 01#A1,A2,A3,A0 02#A1,A2,A0 +counseling 01#A1,A2,A0 +counselor 01#A1,A2,A0 +countdown 01#A1,A0 +countenance 01#A1,A2,A0 +counter 01#A1,A0 +counterbid 01#A1,A2,A0 02#A1,A2,A3,A0 +counterbidder 01#A1,A2,A0 +counterpart 01#A1,A2 +counterpoint 01#A1,A2 +countersuit 01#A1,A2,A3,A0 +counterweight 01#A1,A2 +counting 01#A1,A0 02#A1,A2,A0 +country 01#A1 02#A1 +countryman 01#A1,A0 +countryside 01#A1 +county 01#A1 +coup 01#A1,A0 +couple 01#A1,A3 +coupon 01#A1,A2,A3,A4,A0 +courage 01#A1,A0 +course 01#A1 02#A1,A2,A0 03#A1,A0 +courtesy 01#A1,A2,A0 +courtship 01#A1,A0 +cousin 01#A1,A0 02#A1,A2,A0 +covenant 01#A1,A2,A0 +cover 01#A1,A2,A0 02#A1,A0 +coverage 01#A1,A2,A0 02#A1,A2,A3,A4,A0 03#A1,A0 +covert 01#A1,A3 +coward 01#A1,A0 +cpu 01#A1 +crack 01#A1,A0 02#A1,A2,A0 03#A1,A0 04#A1,A0 +crackdown 01#A1,A2,A3,A0 +cracker 01#A1,A2,A3,A0 +crash 01#A1,A0 +craving 01#A1,A0 +craze 01#A1,A2,A0 +creation 01#A1,A2,A3,A4,A0 +creativity 01#A1,A2,A0 +creator 01#A1,A2,A3,A4,A0 +credence 01#A1,A2 +credential 01#A1,A2,A3 +credibility 01#A1,A2,A3 +credit 01#A1,A2,A0 02#A1,A2,A0 +credit-worthiness 01#A1,A2,A3 +creditor 01#A1,A2,A0 +creed 01#A1,A0 +crest 01#A1 02#A1,A2,A0 +crew 01#A1,A2,A3 +crime 01#A1,A2,A0 +criminal 01#A1,A2,A0 +criminality 01#A1,A2 +crisis 01#A1,A0 +criterion 01#A1,A2,A0 +critic 01#A1,A2,A0 +criticism 01#A1,A2,A0 +critique 01#A1,A0 +crony 01#A1,A2,A0 +crook 01#A1,A2,A0 +crop 01#A1,A0 +crossfire 01#A1,A0 +crowd 01#A1,A2,A3 +crucible 01#A1,A2,A3 +crunch 01#A1,A0 +crusade 01#A1,A0 +crush 01#A1 02#A1,A0 +crusher 01#A1,A2,A3,A0 +crust 01#A1 +cry 01#A1,A2,A0 02#A1,A2,A3 +cryptographer 01#A1,A2,A0 +cub 01#A1,A0 +cue 01#A1,A0 +culmination 01#A1,A2 +culprit 01#A1,A2,A0 +cult 01#A1,A2,A3 +cultivation 01#A1,A0 +culture 01#A1,A2,A3 +cup 01#A1,A3 +curator 01#A1,A2,A3,A0 +curb 01#A1,A2,A3,A4,AM,A0 +cure 01#A1,A2,A3,A0 +current 01#A1,A2 +curriculum 01#A1,A2,A0 +curse 01#A1,A2,A0 +cushion 01#A1,A2,A0 +custody 01#A1,A0 +custom 01#A1,A0 02#A1,A2,A0 +customer 01#A1,A2,A0 +cut 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A4,A0 +cutback 01#A1,A2,A3,A4,A0 +cutoff 01#A1,A2,A3,A4,A0 +cutout 01#A1,A2,A3,A4,A0 +cutter 01#A1,A0 +cutting 01#A1,A2,A3,A4,A0 +cycle 01#A1,A2,A3 +cynicism 01#A1,A0 +czar 01#A1,A0 +dalliance 01#A0 +dam 01#A1,A0 +damage 01#A1,A2,A0 02#A1,A2,A3,A0 +damper 01#A1,A2,A0 +dance 01#A1,A2,A0 02#A1,A0 +dancer 01#A1,A2,A0 +dancing 01#A1,A2,A0 +danger 01#A1,A2 02#A1,A0 +darling 01#A1,A0 +dash 01#A1,A2,A3 02#A1,A0 +date 01#A1,A2,A3 02#A1,A2,A8,A9,A0 +datum 01#A1,A2,A0 +daughter 01#A1,A0 +dawn 01#A1 +day 01#A1,A2,A3 02#A1,A2 03#A1,A2,A3 +dead 01#A1 +deadline 01#A1,A2,A0 +deadlock 01#A1 +deal 01#A1 02#A1,A2,A3,A0 +dealer 01#A1,A0 +dealership 01#A1,A0 +dealing 01#A1,A0 +dean 01#A1,A2,A3,A0 +dearth 01#A1,A3 +death 01#A1,A2 +debacle 01#A1,A0 +debasement 01#A1,A2,A3,A0 +debate 01#A1,A2,A0 +debenture 01#A1,A2,A0 +debt 01#A1,A2,A0 +debtor 01#A1,A2,A3,A0 +debut 01#A1,A2,A0 +decade 01#A1,A2,A3 +decadence 01#A1 +decay 01#A1 +decentralization 01#A1,A0 +deception 01#A1,A2,A0 +decision 01#A1,A2,A3,A0 +deck 01#A1 +declaration 01#A1,A2,A3,A0 +decline 01#A1,A2,A3,A4,AM +decliner 01#A1,A2,A3,A4 +decor 01#A1,A2,A3 +decoration 01#A1,A2,A0 +decorator 01#A1,A2,A0 +decorum 01#A1,A0 +decrease 01#A1,A2,A3,A4,A0 +decree 01#A1,A0 +dedication 01#A1,A2,A0 +deductibility 01#A1,A2,A0 +deed 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A0 +defamation 01#A1,A0 +default 01#A1,A2,A3,A4 +defeat 01#A1,A2,A0 +defect 01#A1,A3,A0 +defection 01#A1,A2,A0 +defendant 01#A1,A2,A3,A0 +defender 01#A1,A2,A0 +defense 01#A1,A2,A3,A0 +deference 01#A1,A0 +defiance 01#A1,A0 +deficiency 01#A1,A2,A3 +deficit 01#A1,A2,A3,A4,A0 +definition 01#A1,A2,A0 02#A1,A2,A0 +deflator 01#A1,A0 +degradation 01#A1,A2,A0 +degree 01#A1,A2 02#A1,A2 03#A1,A2,A3,A0 +delay 01#A1,A2,A0 +delegate 01#A1,A2,A0 +delegation 01#A1,A2,A0 +deletion 01#A1,A2,A0 +deliberation 01#A0 +delight 01#A1,A2,A0 +delinquency 01#A1,A2,A0 +delinquent 01#A1,A0 +delisting 01#A1,A2,A0 +delivery 01#A1,A2,A0 +deluge 01#A1,A3 +delusion 01#A1,A0 +delving 01#A1,A0 +demand 01#A1,A2,A0 +demeanor 01#A1,A0 +dementia 01#A0 +demise 01#A1,A2 +demographics 01#A1,A2 +demonstration 01#A1,A0 02#A1,A2,A0 +demonstrativeness 01#A0 +demonstrator 01#A1,A2,A0 +denial 01#A1,A2,A0 +denigration 01#A1,A0 +denizen 01#A1,A0 +denomination 01#A1,A2 +density 01#A1,A2,A3 +dent 01#A1,A0 +department 01#A1 02#A1 +departure 01#A1,A0 +dependence 01#A1,A0 +dependency 01#A1,A0 +dependent 01#A1,A0 +depiction 01#A1,A2,A0 +depletion 01#A1,A2,A0 +deployment 01#A1,A2,A0 +deportation 01#A1,A2,A3,A0 +deposit 01#A1,A2,A0 +deposition 01#A1,A2,A0 +depositor 01#A1,A2,A0 +depreciation 01#A1 +depressant 01#A1,A2,A3,A4,A0 +depression 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +deprivation 01#A1,A2,A0 +depth 01#A1,A2 +deputy 01#A1,A2,A0 02#A1,A2,A0 +deregulation 01#A1,A0 +deregulaton 01#A1,A0 +dereliction 01#A1,A0 +derision 01#A1,A2,A0 +derivative 01#A1,A2,A3,A4,A0 +derogation 01#A1,A0 +descent 01#A1,A2 02#A1,A2 +description 01#A1,A2,A0 +desecration 01#A1,A2,A0 +design 01#A1,A2,A3,A0 +designation 01#A1,A2,A0 +designee 01#A1,A2,A0 +designer 01#A1,A2,A3,A0 +desire 01#A1,A0 +destination 01#A1,A2 +destiny 01#A1,A2 +destruction 01#A1,A2,A0 +detail 01#A1,A2 +detective 01#A1,A2,A3,A0 +detector 01#A1,A2,A0 +detention 01#A1,A2,A0 +detergent 01#A1,A2,A3,A0 +deterioration 01#A1,A2,A0 +determination 01#A1,A2,A3,A0 +deterrence 01#A1,A2,A3,A0 +deterrent 01#A1,A2,A3,A0 +detoxification 01#A1,A2,A0 +detractor 01#A1,A0 +detriment 01#A1,A2 +devaluation 01#A1,A0 +devastation 01#A1,A2,A0 +developer 01#A1,A2,A3,A4,A0 +developing 01#A1,A2,A3,A4,A0 +development 01#A1,A2 02#A1,A2,A3,A4,A0 +deviation 01#A1,A0 +device 01#A1,A0 +devotee 01#A1,A2,A0 +devotion 01#A1,A2,A0 +diagnosis 01#A1,A2,A0 +diagram 01#A1,A0 +dial 01#A1 +dialogue 01#A1,A2,A0 +dialysis 01#A1,A2,A0 +diameter 01#A1,A2 +diary 01#A1,A0 +dichotomy 01#A1,A2,A0 +dickering 01#A1,A2,A0 +dictate 01#A1,A2,A3,A0 +dictation 01#A1,A0 +dictator 01#A1,A0 +dictatorship 01#A1,A2,A0 +dictum 01#A1,A2,A3,A0 +diet 01#A1,A0 +difference 01#A1,A2,A0 02#A1,A2,A3,A0 +differential 01#A1,A2,A3 +difficulty 01#A1,A2,A3,A0 +dig 01#A1,A2,A0 02#A1,A0 +digging 01#A1,A2,A0 +dignitary 01#A1,A2,A0 +dignity 01#A1,A2,A3 +dilemma 01#A1,A2,A3,A0 +dilution 01#A1,A2,A3,A4,A5,A0 +dimension 01#A1,A2 +diminution 01#A1,A2,A3,A4,A0 +dip 01#A1,A2,A3,A4 02#A1,A2,A0 +diplomacy 01#A1,A0 +diplomat 01#A1,A2,A0 +direction 01#A1,A2,A0 +directive 01#A1,A2,A0 +director 01#A1,A2,A3,A0 +directorship 01#A1,A2,A0 +directory 01#A1,A3,A0 +disability 01#A1,A0 +disadvantage 01#A1,A2,A0 +disaffection 01#A1,A2,A0 +disaffiliation 01#A1,A2,A0 +disagreement 01#A1,A2,A0 +disappearance 01#A1,A0 +disappointment 01#A1,A2,A0 +disapproval 01#A1,A0 +disarray 01#A1 +disaster 01#A1,A0 +disbanding 01#A1,A0 +disbelief 01#A1,A0 +disbursement 01#A1,A2,A3,A0 +discharge 01#A1,A0 +discipline 01#A1,A2,A0 +disclosure 01#A1,A2,A3,A0 +discombobulation 01#A1,A2,A0 +discomfort 01#A1,A0 +discontinuance 01#A1,A0 +discontinuation 01#A1,A0 +discord 01#A1,A2,A0 +discount 01#A1,A2,A3,A4,A0 +discounter 01#A1,A2,A3,A4,A0 +discounting 01#A1,A2,A3,A4,A0 +discouragement 01#A1,A2,A0 +discourse 01#A1,A2,A0 +discovery 01#A1,A2,A3,A0 +discrediting 01#A1,A0 +discrepancy 01#A1,A2,A3 +discretion 01#A1,A0 +discrimination 01#A1,A2,A0 02#A1,A0 +discussion 01#A1,A2,A0 +disease 01#A1,A2 +disfavor 01#A1,A0 +disgorgement 01#A1,A2,A0 +disgrace 01#A1,A2,A0 +disgust 01#A1,A0 +dishonesty 01#A1,A0 +disintegration 01#A1,A0 +dislike 01#A1,A0 +dislocation 01#A1,A0 +disloyalty 01#A1,A0 +dismay 01#A1,A2,A0 +dismissal 01#A1,A2,A0 02#A1,A2,A0 +disobedience 01#A1,A0 +disorder 01#A1,A2,A0 02#A1,A2 +disparity 01#A1,A2,A3 +dispatch 01#A1,A2,A0 +dispatcher 01#A1,A2,A0 +dispensation 01#A1,A2,A0 +displacing 01#A1,A2,A0 +display 01#A1,A2,A0 +disposal 01#A1,A2,A0 02#A1,A2,A0 +disposition 01#A1,A2,A0 02#A1,A2 +dispute 01#A1,A2,A0 +disqualification 01#A1,A2,A0 +disregard 01#A1,A0 +disruption 01#A1,A0 +dissatisfaction 01#A1,A0 +dissection 01#A1,A0 +dissemination 01#A1,A2,A0 +dissent 01#A1,A0 +dissenter 01#A1,A0 +disservice 01#A1,A2,A0 +dissident 01#A1,A2,A0 +dissolution 01#A1,A2,A3,A0 +dissonance 01#A1,A2 +distance 01#A1,A2,A3 +distaste 01#A1,A0 +distinction 01#A1,A2,A3,A0 +distinctiveness 01#A1,A2 +distortion 01#A1,A0 +distraction 01#A1,A2,A0 +distress 01#A1,A2,A0 +distributer 01#A1,A2,A0 +distribution 01#A1,A2,A0 +distributorship 01#A1,A2,A0 +district 01#A1,A2,A3 +districting 01#A1,A2,A0 +distrust 01#A1,A0 +disturbance 01#A1,A2,A0 +dive 01#A1,A2,A0 02#A1,A2,A3,A4 +divergence 01#A2,A0 +diversification 01#A1,AA,A2,A0 +diversion 01#A1,A2,A3,A0 +diversity 01#A1,A2,A3 +divestiture 01#A1,A2,A0 +divide 01#A1,A2,A0 +dividend 01#A1,A2,A3,A0 +division 01#A1,A2 02#A1,A2,A0 +divisiveness 01#A1,A0 +doctor 01#A1,A2 02#A1,A2,A0 +doctorate 01#A1,A2,A0 +doctrine 01#A1,A2,A3,A0 +document 01#A1,A0 +documentary 01#A1,A0 +documentation 01#A1,A0 +doer 01#A1,A2,A3,A4,A0 +doldrums 01#A1,A2 +dollar 01#A1,A3 +domain 01#A1,A0 +dominance 01#A1,A0 +domination 01#A1,A0 +dominion 01#A1,A0 +don 01#A1,A2,A0 +donation 01#A1,A2,A0 +donor 01#A1,A2,A0 +doomsayer 01#A1,A0 +door 01#A1 +dormitory 01#A1,A2,A0 +dosage 01#A1,A2,A0 +dose 01#A1,A3 +doubling 01#A1,A2,A3,A4,A0 +doubt 01#A1,A0 +doubter 01#A1,A0 +down 01#A1,A2,A3,A4,A0 +downdraft 01#A1,A2,A3,A4,AM +downfall 01#A1,A2 +downgrade 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +downgrading 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +downside 01#A1 +downsizing 01#A1,A2,A3,A4,A0 +downtown 01#A1 +downtrend 01#A1,A2,A3,A4,AM +downturn 01#A1,A2,A3,A4,AM +dozen 01#A1,A3 +draft 01#A1,A0 +drag 01#A1,A2,A0 +drain 01#A1,A2,A0 +drama 01#A1,A0 02#A1,A2 +dramatization 01#A1,A0 +drawback 01#A1,A2 +drawing 01#A1,A2,A0 02#A1,A2,A0 +dream 01#A1,A0 +drenching 01#A1,A2,A0 +dress 01#A1,A0 +dribble 01#A1,A3 +drift 01#A1,A2,A3 +drilling 01#A1,A2,A3,A4,A0 +drink 01#A1,A2,A0 +drinker 01#A1,A2,A0 +drinking 01#A1,A2,A0 +drive 01#A1,A0 02#A1,A0 03#A1,A0 04#A1,A0 +driver 01#A1,A2,A0 +driving 01#A1,A0 02#A1,A2,A0 +drop 01#A1,A2,A3,A4,A5 02#A1,A3 +drop-off 01#A1,A2,A3,A4,AM,A0 +dropout 01#A1,A0 +dropper 01#A1,A0 +drought +drubbing 01#A1,A2,A0 +drug 01#A1,A2,A3 +drumbeat 01#A1,A2,A3,A0 +drumroll 01#A1,A3 +due 01#A1,A2,A3,A0 +duel 01#A1,A2,A0 +duke 01#A1,A3,A0 +duplicity 01#A0 +durability 01#A1,A2 +duration 01#A1,A2 +dust-up 01#A1,A2,A0 +duty 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +dweller 01#A1,A0 +dwelling 01#A1,A0 +dynamic 01#A1,A2 +dynamism 01#A1,A2 +eagerness 01#A1,A0 +ear 01#A1 02#A1,A0 +earner 01#A1,A2,A3,A0 +earning 01#A1,A2,A3,A0 +ease 01#A1 02#A1,A0 +easing 01#A1,A0 +eater 01#A1,A0 +eavesdropping 01#A1,A0 +ebb 01#A1 +echelon 01#A1,A2,A0 +echo 01#A1,A0 +economics 01#A1,A2,A3 +economist 01#A1,A2,A3,A0 +economy 01#A1,A2,A3 +edge 01#A1 02#A1,A2,A0 03#A1,A0 +edition 01#A1,A0 +editor 01#A1,A2,A3,A0 +editorial 01#A1,A0 +education 01#A1,A2,A0 +eduction 01#A1,A2,A0 +effect 01#A1,A2,A3,A0 +effectiveness 01#A1 02#A1 +efficiency 01#A1,A0 +effort 01#A1,A2,A0 +effrontery 01#A1,A0 +egalitarianism 01#A1,A2 +egg 01#A1,A3,A0 +ego 01#A1,A2 +eighth 01#A1,A3 +elan 01#A1,A2 +elder 01#A1,A0 +election 01#A1,A2,A0 +electrolysis 01#A1,A0 +element 01#A1 02#A1 03#A1,A2,A3 +elimination 01#A1,A2,A0 +elitist 01#A1,A0 +eloquence 01#A1,A0 +emasculation 01#A1,A0 +embargo 01#A1,A0 +embarrassment 01#A1,A2,A0 +embassy 01#A1,A2,A0 +embezzlement 01#A1,A2,A0 +emblem 01#A1,A2 +emergence 01#A1,A0 +emergency 01#A1,A0 +emigration 01#A1,A2,A0 +emissary 01#A1,A0 +emission 01#A1,A0 +emotion 01#A1,A0 +emphasis 01#A1,A2,A0 +empire 01#A1,A2,A3 +employee 01#A1,A2,A0 +employer 01#A1,A2,A0 +employment 01#A1,A2,A0 +enactment 01#A1,A2,A0 +encounter 01#A1,A0 +encouragement 01#A1,A2,A0 +end 01#A1,A2,AM,A0 02#A1,A2 03#A1,A0 +endangerment 01#A1,A0 +ending 01#A1,A2,AM,A0 +endorsement 01#A1,A2,A0 +endowment 01#A1,A2,A0 +enemy 01#A1,A2,A0 +energy 01#A1,A2 +enforcement 01#A1,A0 +enforcer 01#A1,A0 +engagement 01#A1,A2,A3,A0 +engine 01#A1 02#A1,A0 +engineer 01#A1,A2,A3,A0 +engineering 01#A1,A4,A0 +engorgement 01#A1,A2,A3,A4,A0 +enhancement 01#A1,A2,A3,A0 +enjoyment 01#A1,A0 +enlightenment 01#A1,A2,A3,A0 +enrollee 01#A1,A2,A0 +enrollment 01#A1,A2,A0 +enterprise 01#A1,A0 +entertainment 01#A1,A2,A0 +enthusiasm 01#A1,A0 +enthusiast 01#A1,A0 +entirety 01#A1,A2 +entitlement 01#A1,A2,A0 +entrance 01#A1,A0 +entrant 01#A1,A2 +entrenchment 01#A1,A2,A0 +entry 01#A1,A0 02#A1,A2,A3,A0 +environment 01#A1,A2,A3 02#A1 +envoy 01#A1,A0 +envy 01#A1,A2,A0 +envy-quotient 01#A1,A2 +epicenter 01#A1,A0 +epidemic 01#A1,A2 +epidemiologist 01#A1,A2,A0 +episode 01#A1,A0 02#A1,A0 +epsiode 01#A1,A0 +equality 01#A1,A2,A3 +equation 01#A1,A2,A0 +equilibrium 01#A1,A2,A3 +equipment 01#A1,A2,A0 +equity 01#A1,A2,A0 +equivalent 01#A1,A2 +era 01#A1 +erasure 01#A1,A2,A3,A0 +erosion 01#A1,A0 +error 01#A1,A2,A0 +eruption 01#A1,A2 +escalation 01#A1,A0 +escape 01#A1,A0 +escort 01#A1,A3,A4,A0 +escrow 01#A1,A2,A3,A0 +espionage 01#A1,A0 +espousal 01#A1,A0 +essay 01#A1,A3,A0 +essence 01#A1,A0 +establishment 01#A1,A2,A3 02#A1,A2,A3,A0 +esteem 01#A1,A2,A0 +estimate 01#A1,A2,A0 +estimation 01#A1,A2,A0 +estimator 01#A1,A2,A0 +ethics 01#A1,A2 +ethos 01#A1,A2 +etiquette 01#A1,A2,A3,A0 +euphoria 01#A1,A0 +evacuation 01#A1,A2,A0 +evader 01#A1,A0 +evaluation 01#A1,A0 +evaporation 01#A1,A0 +evasion 01#A1,A0 +eve 01#A1 +evening 01#A1,A2,A3 +event 01#A1,A2 +evidence 01#A1,A2,A0 +evolution 01#A1,A2,A3 +examination 01#A1,A0 +examiner 01#A1,A0 +example 01#A1,A2 +exasperation 01#A1,A0 +excellence 01#A1,A2 +excerpt 01#A1,A2,A0 +excess 01#A1,A2,A0 +exchange 01#A1,A2,A3,A4,A0 02#A1,A2,A0 +excision 01#A1,A2,A0 +excitement 01#A1,A2,A0 +exclusion 01#A1,A2,A0 +exclusivity 01#A1,A2,A3 +excursion 01#A1,A0 +excursus 01#A1,A0 +excuse 01#A1,A2,A0 +execution 01#A1,A2,A0 02#A1,A2,A3,A0 +executive 01#A1,A2,A3,A0 +exemplar 01#A1,A0 +exemption 01#A1,A2,A0 +exercise 01#A1,A0 02#A0 +exhaustion 01#A1,A0 +exhibit 01#A1,A2,A0 +exhibition 01#A1,A2,A0 +exile 01#A1,A2,A3,A0 +existence 01#A1,A2 +exit 01#A1,A0 +exodus 01#A1,A2,A0 +expansion 01#A1,A2,A3,A4,A0 +expansionism 01#A1,A2,A3,A4,A0 +expectancy 01#A1,A2,A0 +expectation 01#A1,A0 +expedition 01#A1,A2,A3 +expenditure 01#A1,A2,A3,A4,A0 +expense 01#A1,A2,A0 +experience 01#A1,A0 +experiment 01#A1,A2,A0 +experimentation 01#A1,A2,A0 +expert 01#A1,A2,A0 +expertise 01#A1,A0 +expiration 01#A1,AM +explanation 01#A1,A2,A0 +exploit 01#A1,A2,A3,A4,A0 +exploration 01#A1,A0 +explosion 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +explusion 01#A1,A2,A0 +export 01#A1,A2,A0 +exporter 01#A1,A2,A0 +exposure 01#A1,A2,A0 +expression 01#A1,A2,A0 +extension 01#A1,A2,A3,A4,A0 +extent 01#A1,A2,A3,A4,A0 +extinction 01#A1 +extortion 01#A1,A2,A0 +extract 01#A1,A2,A0 +extraction 01#A1,A2,A0 02#A1,A2,A3 +extradition 01#A1,A2,A3,A0 +extrusion 01#A1,A2,A0 +exuberance 01#A0 +eye 01#A1,A0 02#A1 +eyeball 01#A1 +eyebrow 01#A1 +eyewitness 01#A1,A0 +fabric 01#A1,A0 +fabrication 01#A1,A2,A3,A0 +fabricator 01#A1,A2,A3,A0 +facade 01#A1 +face 01#A1,A2 02#A1 03#A1,A0 04#A1 05#A1,A0 06#A1,A0 +facsimile 01#A1,A2,A0 +fact 01#A1 +faction 01#A1,A2,A3 +factor 01#A1,A2,A3 02#A1,A2 +factory 01#A1,A2,A3,A0 +faculty 01#A1,A2,A3,A0 +fad 01#A1,A2 +failure 01#A1,A0 +fairness 01#A1,A2,A3,A0 +faith 01#A1,A0 +fall 01#A1,A2,A3,A4,AM 02#A1 +falloff 01#A1,A2,A3,A4,AM +fallout 01#A1,A2,A3,A0 +fame 01#A1,A2,A3,A0 +familiarity 01#A1,A0 +familiarization 01#A1,A2,A0 +family 01#A1,A2,A3 02#A1 +famine +fan 01#A1,A2,A0 +fancy 01#A1,A0 +fantasy 01#A1,A0 +fare 01#A1,A2,A0 02#A1,A0 +farm 01#A1,A2,A3,A0 +farmer 01#A1,A0 +farming 01#A1,A0 +fashion 01#A1,A2 02#A1,A0 +fatality 01#A1 +fate 01#A1,A2 +father 01#A1,A0 02#A1,A2,A0 +fault 01#A1,A2,A0 02#A1,A2,A0 +favor 01#A1,A0 02#A1,A2,A0 +favorite 01#A1,A2,A0 +favoritism 01#A1,A2,A0 +fax 01#A1,A0 +fear 01#A1,A0 +feasibility 01#A1 +feat 01#A1,A2,A3,A4,A0 +feature 01#A1,A2,A3 02#A1,A0 +fee 01#A1,A2,A3,A0 +feedback 01#A1,A2,A0 +feeding 01#A1,A2,A0 +feel 01#A1,A2,A0 02#A1,A2,A0 +feeler 01#A1 +feeling 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2 +fellow 01#A1,A2,A0 02#A1,A2,A0 +felony 01#A1,A2,A0 +femininity 01#A1 +fertility 01#A1 +fertilization 01#A1,A0 +fervor 01#A1,A0 +fest 01#A1,A0 +festival 01#A1,A0 +festivity 01#A1,A2,A0 +fetish 01#A1,A0 +fetus 01#A1,A0 +feud 01#A1,A2,A0 +fever 01#A1,A0 +fiancee 01#A1,A0 +fiber 01#A1,A3 +fickleness 01#A1 +fiddling 01#A1,A0 +fidelity 01#A1,A2 +field 01#A1 02#A1 03#A1,A2,A3 +fifth 01#A1,A3 +fight 01#A1,A2,A0 +fighter 01#A1,A2,A0 +fighting 01#A1,A2,A0 +figure 01#A1,A0 02#A1,A2,A3,A0 03#A1,A2 04#A1,A0 +file 01#A1,A2,A3,A4,A0 +filers 01#A1,A2,A3,A4,A0 +filibuster 01#A1,A2,A0 +filing 01#A1,A2,A3,A4,A0 +filling 01#A1,A2,A0 +film 01#A1,A0 02#A1,A2,A0 +filter 01#A1,A2,A0 +finance 01#A1,A2,A3 02#A1,A2,A0 +financer 01#A1,A2,A3,A0 +financier 01#A1,A2,A3,A0 +financing 01#A1,A2,A3,A0 +find 01#A1,A2,A3,A0 +finder 01#A1,A2,A3,A0 +finding 01#A1,A2,A3,A0 +fine 01#A1,A2,A3,A0 +finger 01#A1 +fingerprint 01#A1,A2 +finish 01#A1,AM 02#A1,A2,A0 +fire 01#A1 02#A1,A2,A0 03#A1 +firefighter 01#A1,A2,A0 +firefighting 01#A1,A2,A0 +firing 01#A1,A2,A0 +firmness 01#A1,A2 +fishing 01#A1,A0 +fit 01#A1,A2,A3,A0 02#A1,A3 03#A1,A0 +fitness 01#A1,A2 +fix 01#A1,A2,A0 +flag 01#A1,A2,A0 +flagship 01#A1,A0 02#A1 +flair 01#A1,A0 +flap 01#A1,A2,A0 02#A1,A3 +flash 01#A1,A2,A3 02#A1,A2,A0 +flashback 01#A1,A0 +flat 01#A1,A3 +flatness 01#A1,A2 +flavor 01#A1,A2 +flaw 01#A1,A0 +fledgling 01#A1,A2 +fleet 01#A1,A3 +flexibility 01#A1,A0 +flier 01#A1,A2,A0 +flight 01#A1,A2,A3,A4,A0 02#A1,A0 +flightiness 01#A1 +fling 01#A1,A2,A3,A4,A0 +flip-flop 01#A1,A2,A3,A0 +flirtation 01#A1,A0 +flock 01#A1,A3 +flood 01#A1,A2,A3 +flooding 01#A1,A2,A0 +floor 01#A1 02#A1,A2,A3 +flotation 01#A1,A2,A0 +flotilla 01#A1,A2,A3 +flow 01#A1,A2,A3 +fluctuation 01#A1,A2,A3,A4,A5 +flurry 01#A1,A3 +focus 01#A1,A2,A0 +foe 01#A1,A0 +foldability 01#A1,A2 +folding 01#A1,A0 +folio 01#A1,A2,A3 +follower 01#A1,A2,A0 +following 01#A1,A2,A0 +folly 01#A1,A0 +fondness 01#A1,A0 +food 01#A1,A2,A0 +fooling 01#A1,A2,A0 +foot 01#A1,A3 02#A1 +footage 01#A1,A0 +foothold 01#A1,A2,A0 +footing 01#A1,A2,A3 +footnote 01#A1,A2,A0 +foray 01#A1,A2,A0 +forbearance 01#A1,A2,A3,A0 +force 01#A1,A2,A3 02#A1,A2,A3 03#A1,A2,A0 +forcefulness 01#A1,A0 +forecast 01#A1,A3,A0 +forecaster 01#A1,A0 +forecasting 01#A1,A0 +foreclosure 01#A1,A0 +forefront 01#A1,A0 +forehead 01#A1 +foreman 01#A1,A2,A0 +forerunner 01#A1,A2,A0 +forfeiture 01#A1,A2,A0 +forgery 01#A1,A0 +forgiveness 01#A1,A2,A0 +form 01#A1 +formality 01#A1,A0 +format 01#A1,A2 +formation 01#A1,A2,A3,A4,A0 +formula 01#A1,A0 +formulation 01#A1,A2,A3,A4,A0 +fortune 01#A1,A2,A3 +forum 01#A1 +fouling 01#A1,A2,A0 +foundation 01#A1,A2,A0 02#A1,A2,A3 +founder 01#A1,A2,A0 +founding 01#A1,A2,A0 +fourth 01#A1,A3 +fracas 01#A1,A2,A0 +fraction 01#A1,A3 +fragility 01#A1,A2 +fragment 01#A1 +frailty 01#A1,A2 +frame 01#A1 02#A1 +framer 01#A1,A0 +framework 01#A1 +franc 01#A1,A3 02#A1 +franchise 01#A1,A2,A0 +franchisee 01#A1,A2,A0 +franchiser 01#A1,A2,A0 +fraud 01#A1,A0 +freedom 01#A1,A2,A3,A0 +freeing 01#A1,A2,A3,A0 +freeze 01#A1,A2,A0 +freezing 01#A1,A2,A0 +freight 01#A1,A0 +frenzy 01#A1,A3,A0 +frequency 01#A1,A2,A3 +freshman 01#A1 +friction 01#A1,A2,A0 +friend 01#A1,A0 02#A1,A2,A0 +friendliness 01#A1,A2,A0 +friendship 01#A1,A2,A0 +front 01#A1 02#A1,A2 03#A1,A2,A0 04#A1,A0 +frontier 01#A1,A2 +frugality 01#A1,A0 +fruit 01#A1,A2,A3,A0 +fruition 01#A1 +frustration 01#A1,A0 +fuel 01#A1,A2,A0 +fugitive 01#A1,A2 +fulmination 01#A1,A2,A0 +fume 01#A1,A2,A3,A0 +fun 01#A1,A0 +function 01#A1,A0 02#A1,A0 03#A1,A0 +functionary 01#A1,A0 +functioning 01#A1,A0 +fund 01#A1,A2,A3,A0 +fundamental 01#A1,A2,A3 +fundamentalism 01#A1,A0 +funding 01#A1,A2,A0 +furor 01#A1,A0 02#A1,A2 +fury 01#A1,A0 +fusillade 01#A1,A2,A0 +fusing 01#A1,A2,A3,A0 +fusion 01#A1,A2,A3,A0 +futility 01#A1 +future 01#A1,A3 +gadget 01#A1 +gain 01#A1,A2,A3,A4 02#A1,A2,A3,A0 +gainer 01#A1,A2,A3,A4,AM +gala 01#A1,A2,A0 +gallery 01#A1,A2,A3 +gallon 01#A1,A3 +gamble 01#A1,A0 +gambler 01#A1,A0 +game 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 04#A1,A0 05#A1 +gamut 01#A1,A2,A3,A4,A5,A0 +gap 01#A1,A2,A3,A0 +gate 01#A1 +gateway 01#A2,A3,A4 +gauge 01#A1,A2,A3,A0 +gear 01#A1 +gene 01#A1,A2,A0 +general 01#A1,A2,A3,A0 02#A1,A0 +generalization 01#A1,A0 +generating 01#A1,A2,A3,A4,A0 +generation 01#A1,A2,A3,A4,A0 +generator 01#A1,A2,A3,A4,A0 +genesis 01#A1,A2,A0 +genius 01#A1,A0 02#A1,A2,A3 +genocide 01#A1,A2,A0 +genre 01#A1,A0 +geography 01#A1,A2 +gerrymandering 01#A1,A2,A0 +gesture 01#A1,A2,A0 +getter 01#A1,A2,A3,A4,A0 +ghost 01#A1,A0 +gift 01#A1,A2,A0 +gig 01#A1,A2,A0 +gimmick 01#A1,A0 +gimmickry 01#A1,A0 +girlfriend 01#A1,A0 +gist 01#A1,A0 +giveaway 01#A1,A2,A0 +glade 01#A1,A2,A3 +glamour 01#A1,A2,A3 +glance 01#A1,A0 +glare 01#A1,A0 +glee 01#A1,A0 +glimmer 01#A1 +glimpse 01#A1,A0 +glitch 01#A1,A0 +globalization 01#A1,A0 +gloom 01#A1,A0 +glory 01#A1 +glue 01#A1,A2,A0 +glut 01#A1,A3 +go +go-ahead 01#A1,A2,A0 +goal 01#A1,A0 +goatee 01#A1 +goer 01#A1,A3,A4,AM,A0 +goings-on 01#A1,A2 +good 01#A1,A2 +goodness 01#A1,A2,A3 +gospel 01#A1,A0 +gossip 01#A1,A0 +governance 01#A1,A0 +governing 01#A1,A0 +government 01#A1,A2,A3,A0 +governor 01#A2,A3,A0 +grab 01#A1,A0 +grade 01#A1,A2 +grader 01#A1,A3,A0 +graduate 01#A1,AA,A2,A0 +graduation 01#A1,AA,A2,A0 +grain 01#A1,A2,A3 02#A1,A3 +gram 01#A1,A3 +grandchild 01#A1,A3,A0 +grandee 01#A1,A2,A0 +grandfather 01#A1,A3,A0 +grandkid 01#A1,A0 +grandmother 01#A1,A0 +grandson 01#A1,A0 +grant 01#A1,A2,A0 +grasp 01#A1,A0 +gratuity 01#A1,A2,A0 +grave 01#A1,A0 +gravity 01#A1,A2,A3 +graying 01#A1,A0 +grazing 01#A1,AA,A0 +greatness 01#A1,A2 +greed 01#A1,A2,A3,A0 +greening 01#A1,A0 +gridlock 01#A1,A2,A0 02#A1,A2,A0 +grievance 01#A1,A2,A3,A0 02#A1,A2,A0 +grinder 01#A1,A2,A0 +grip 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +ground 01#A1,A0 02#A1,A2,A0 +grounding 01#A1,A2,A0 +groundwork 01#A1,A2,A0 +group 01#A1,A2,A3 +grove 01#A1,A2,A3 +grower 01#A1,A2,A0 +growing 01#A1,A2,A0 +growth 01#A1,A2,A3,A4,A5,A0 02#A1,A2,A0 +guarantee 01#A1,A2,A0 +guarantor 01#A1,A2,A0 +guard 01#A1,A2,A0 +guardian 01#A1,A2,A0 +guerrilla 01#A1,A2,A0 +guess 01#A1,A0 +guesswork 01#A1,A0 +guest 01#A1,A0 +guidance 01#A1,A2,A3,A0 +guide 01#A1,A2,A3,A0 +guideline 01#A1,A2,A0 +guidepost 01#A1,A2,A3,A0 +guilder 01#A1,A3 +guilt 01#A1,A0 02#A1,A2,A0 +guise 01#A1,A0 +guru 01#A1,A0 +gut 01#A1,A0 02#A1,A2,A0 +gyration 01#A1,A0 +habit 01#A1,A0 +habitat 01#A1 +hackles 01#A1,A0 +hail 01#A1,A3 +hairdresser 01#A1,A0 +half 01#A1,A3 +hall 01#A1 +hallmark 01#A1,A0 +halt 01#A1,A2,A0 +hand 11#A1,A0 01#A1,A0 12#A1,A0 02#A1 03#A1,A2 04#A1,A2,A0 05#A1,A0 06#A1,A2,A3 07#A1 08#A1,A2,A0 09#A1,A0 10#A1,A0 +handbook 01#A1,A2,A3,A0 +handful 01#A1,A3 +handler 01#A1,A2,A3,A0 +handling 01#A1,A2,A3,A0 +handout 01#A1,A2,A0 02#A1,A0 +handwriting 01#A1,A2 +hangover 01#A1,A2 +hank 01#A1,A3 +happening 01#A1,A2 +happiness 01#A1,A0 +harangue 01#A1,A2,A0 +harassment 01#A1,A2,A0 +harbinger 01#A1,A0 +hardship 01#A1,A2,A3,A0 +harm 01#A1,A2,A0 +harmony 01#A1,A2,A0 +harshness 01#A1,A2 +harvest 01#A1,A0 +harvesting 01#A1,A0 +haste 01#A1,A2,A0 +hatred 01#A1,A0 +haulage 01#A1,A2,A3,A4,A0 +hauler 01#A1,A2,A3,A4,A0 +hauteur 01#A1,A2 +haven 01#A1,A2,A3,A0 +havoc 01#A1,A2,A0 +hazard 01#A1,A0 +head 01#A1,A2,A3,A0 02#A1,A3 03#A1,A2 +headache 01#A1,A2,A3,A0 +headline 01#A1,A2,A0 +headquarters 01#A1,A2,A0 +healing 01#A1,A0 +health 01#A1,A2 +heap 01#A1,A3 +hearing 01#A1,A2,A0 02#A1,A2,A3 +heart 01#A1,A0 02#A1 03#A1,A0 04#A1,A0 05#A1 +heartland 01#A1,A3,A0 +heat 01#A1,A2,A3 02#A1,A2,A0 03#A1,A2,A0 +heater 01#A1,A2,A0 +heating 01#A1,A2,A0 +heavy-handedness 01#A1 +hedge 01#A1,A2,A3,A0 +heel 01#A1 02#A1,A2 +height 01#A1,A2,A3 +heir 01#A1,A0 +heist 01#A1,A2,A0 +helm 01#A1,A0 +help 01#A1,A2,A0 +hemorrhaging 01#A1,A0 +herd 01#A1,A2,A3 +heritage 01#A1,A2,A0 +hero 01#A1,A2,A0 +heroine 01#A1,A2,A0 +hesitation 01#A1,A0 +heyday 01#A1 +hideaway 01#A1,A2,A3,A0 +hideout 01#A1,A2,A3,A0 +hider 01#A1,A2,A0 +hierarchy 01#A1,A2,A3 +high 01#A1,A2 +highway 01#A1,A0 +hike 01#A1,A2,A3,A4,AM +hiking 01#A1,AA,A0 +hill 01#A1,A3 +hint 01#A1,A2,A0 +hiring 01#A1,A2,A0 +historian 01#A1,A2,A0 +history 01#A1,A3,A0 02#A1,A0 +hit 01#A1,A2,A0 02#A1,A2,A3,A0 03#A1,A2,A0 04#A1,A2,A0 +hitch 01#A1,A0 02#A1,A2,A0 +hitter 01#A1,A2,A0 +hoard 01#A1,A3,A0 +hoarder 01#A1,A0 +hodgepodge 01#A1,A2,A3 +hoisting 01#A1,A2,A0 +hold 01#A1,A2,A3,A0 02#A1 +holder 01#A1,A2,A3,A0 +holding 01#A1,A2,A3,A0 02#A1,A2 +holdout 01#A1,A2 +holdup 01#A1,A0 +hole 01#A1 02#A1,A2,A0 +holiday 01#A1,A0 +homage 01#A1,A2,A0 +home 01#A1,A0 02#A1,A2,A0 03#A1,A2 +homecoming 01#A1,A2,A3,A4,AM +homeland 01#A1,A0 +hometown 01#A1,A0 +homework 01#A1,A2,A0 +honesty 01#A1,A0 +honor 01#A1,A2,A0 +hook-up 01#A1,A2,A0 +hope 01#A1,A0 +horde 01#A1,A3 +horizon 01#A1,A0 02#A1,A0 +hormone 01#A1,A2,A0 +horn 01#A1 +horror 01#A1 +horticulturist 01#A1,A0 +hospitality 01#A1,A2,A0 +hospitalization 01#A1,A0 +host 01#A1,A0 02#A1 +hostage 01#A1,A0 +hostility 01#A1,A2,A3 +hour 01#A1,A2,A3 +house 01#A1,A0 02#A1 03#A1 04#A1,A2,A0 +housework 01#A1,A2,A3,A4,A0 +housing 01#A1 02#A1,A0 +howl 01#A1,A3 02#A1,A2,A0 +hub 01#A1,A2,A3 +hug 01#A1,A0 +humility 01#A1,A2,A0 +humor 01#A1,A2,A0 +humorist 01#A1,A2,A0 +hunch 01#A1,A2,A0 +hundreds 01#A1,A3 +hunger 01#A1,A0 +hunk 01#A1,A3 +hunt 01#A1,A0 +hunter 01#A1,A0 +hunting 01#A1,A0 +hurdle 01#A1,A2,A0 +hurricane +hurry 01#A1,A2,A0 +husband 01#A1,A0 +husk 01#A1 +hybrid 01#A1,A2,A3,A0 +hype 01#A1,A0 +hyping 01#A1,A2,A0 +hypocrisy 01#A1,A0 +idea 01#A1,A2,A0 02#A1,A2,A0 +ideal 01#A1,A2,A0 +idealism 01#A1,A0 +identification 01#A1,A2,A0 02#A1,A2,A0 +identity 01#A1,A2,A3 +ideology 01#A1,A0 +idiocy 01#A1,A0 +idling 01#A1,A0 +ignoramus 01#A1,A0 +ignorance 01#A1,A0 +ill 01#A1,A2,A3 +illiquidity 01#A1,A2 +illiteracy 01#A0 +illness 01#A1,A2 +illogic 01#A1,A0 +illusion 01#A1 +illustration 01#A1,A2,A0 +image 01#A1,A2,A0 +imagery 01#A1,A0 +imagination 01#A1,A2,A0 +imbalance 01#A1,A2,A3,A0 +imitation 01#A1,A0 +immaturity 01#A1,A0 +immigrant 01#A1,A2,A0 +immigration 01#A1,A2,A0 +immorality 01#A1,A0 +immunity 01#A1,A0 +immunologist 01#A1,A2,A0 +impact 01#A1,A0 02#A1,A2 +impairment 01#A1,A0 +impasse 01#A1,A2,A0 +impatience 01#A1,A0 +impeachment 01#A1,A2,A0 +impediment 01#A1,A2,A3,A0 +imperative 01#A1,A0 +imperfection 01#A1,A3,A0 +impersonation 01#A1,A0 +impersonator 01#A1,A0 +impetus 01#A1,A2,A0 +implant 01#A1,A2,A0 +implantation 01#A1,A2,A0 +implementation 01#A1,A2,A0 +implication 01#A1,A2,A0 +import 01#A1,A2,A0 02#A1 +importance 01#A1,A2,A3,A0 +importer 01#A1,A2,A0 +imposition 01#A1,A2,A0 +impression 01#A1,A2,A0 02#A1,A2,A0 +imprisonment 01#A1,A2,A3,A0 +improbability 01#A1,A2 +improvement 01#A1,A0 +improvisation 01#A1,A0 +impulse 01#A1 02#A1,A0 +impunity 01#A1,A2,A0 +inability 01#A1,A0 +inaccuracy 01#A1,A2,A0 +inactivation 01#A1,A0 +inadequacy 01#A1,A2 +inadvertence 01#A0 +inattention 01#A1,A0 +inauguration 01#A1,A2,A0 +incentive 01#A1,A2,A3,A0 +inception 01#A1,A2,A0 +incest 01#A1,A0 +inch 01#A1,A3 +incidence 01#A1 +incident 01#A1,A2 +incinerator 01#A1,A0 +inclination 01#A1,A0 +inclusion 01#A1,A2,A0 +income 01#A1,A2,A3,A0 +incompatibility 01#A1,A2,A0 +incompetence 01#A1,A0 +incompetency 01#A1,A0 +inconsistency 01#A1,A2,A0 +inconvenience 01#A1,A2 +increase 01#A1,A2,A3,A4,A0 +increment 01#A1,A2 +incrimination 01#A1,A2,A0 +incursion 01#A1,A2,A0 +indebtedness 01#A1,A2,A3,A0 +indecisiveness 01#A0 +independence 01#A1,A2 +index 01#A1,A2,A3,A0 +indexation 01#A1,A2,A0 +indexing 01#A1,A2,A0 +indication 01#A1,A2,A0 +indicator 01#A1,A2,A0 +indictment 01#A1,A2,A0 +indifference 01#A1,A0 +indignity 01#A1,A0 +indirectness 01#A1 +indispensability 01#A1,A2,A0 +inducement 01#A1,A0 +indulgence 01#A1,A2,A0 +industrialization 01#A1 +industry 01#A1 +ineffectiveness 01#A1,A2 +inefficiency 01#A1,A2 +ineptitude 01#A1,A0 +inequality 01#A1,A2,A3 +inequity 01#A1,A2 +inertia 01#A1,A0 +inevitability 01#A1 +inexperience 01#A1,A0 +infancy 01#A1,A2,A0 +infantry 01#A1,A0 +infection 01#A1,A2,A3,A0 +inference 01#A1,A2,A0 +infertility 01#A1 +infestation 01#A1,A0 +infidelity 01#A2,A0 +infighting 01#A1,A2,A0 +inflation 01#A1,A2,A3,A4,A0 +inflow 01#A1,A2,A3 +influence 01#A1,A2,A0 +influx 01#A1,A2,A3 +information 01#A1,A2,A0 +infraction 01#A1,A0 +infrastructure 01#A1,A2 +infringement 01#A1,A0 +ingestion 01#A1,A0 +ingredient 01#A1 +inheritance 01#A1,A2,A0 +inheritor 01#A1,A2,A0 +initiation 01#A1,A2,A0 +initiative 01#A1,A0 +initiator 01#A1,A2,A0 +injunction 01#A1,A2,A0 +injury 01#A1,A2,A0 +injustice 01#A1,A2,A3,A0 +inkling 01#A1,A0 +inmate 01#A1,A0 +inning 01#A1 +innocence 01#A1,A0 +innovation 01#A1,A0 +input 01#A1,A2,A0 02#A1,A0 +inquiry 01#A1,A0 +insensitivity 01#A1,A0 +insert 01#A1,A2,A0 +inside 01#A1 +insider 01#A1,A0 +insight 01#A1,A2,A0 +insistence 01#A1,A0 +insolvency 01#A1 +inspection 01#A1,A2,A0 +inspector 01#A1,A2,A3,A4,A0 +inspiration 01#A1,A2,A0 +instability 01#A1,A2 +installation 01#A1,A2,A0 +installment 01#A1,A0 +instance 01#A1,A3 +institution 01#A1,A2,A3 02#A1,A2,A0 +instruction 01#A1,A2,A0 +instructor 01#A1,A2,A0 +instrument 01#A1,A0 +insubordination 01#A1,A0 +insulation 01#A1,A2,A3,A0 +insulator 01#A1,A2,A3,A0 +insult 01#A1,A0 +insurance 01#A1,A2,A3,A4,A0 +insurer 01#A1,A2,A3,A4,A0 +insurgent 01#A1,A0 +integration 01#A1,A2,A0 +integrity 01#A1,A2 +intelligence 01#A1,A2 02#A1,A2,A0 +intensifier 01#A1,A2,A0 +intensity 01#A1,A2,A3 +intent 01#A1,A2,A0 +intention 01#A1,A2,A0 +interaction 01#A1,A0 +interdiction 01#A1,A0 +interest 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1 05#A1,A2 +interference 01#A1,A0 +interior 01#A1 +internationalization 01#A1,A0 +internment 01#A1,A2,A0 +interpretation 01#A1,A2,A0 +interpreter 01#A1,A2,A0 +interrogation 01#A1,A2,A0 +interrogator 01#A1,A2,A0 +interruption 01#A1,A0 +interval 01#A1,A2 +intervention 01#A1,A0 +interview 01#A1,A2,A0 +interviewer 01#A1,A2,A0 +intimacy 01#A1,A0 02#A1,A2 +intimidation 01#A1,A0 +intonation 01#A1,A0 +intoxication 01#A1,A2,A0 +introduction 01#A1,A0 02#A1,A2,A0 +intruder 01#A1,A2,A0 +intrusion 01#A1,A2,A0 +intuition 01#A1,A2,A0 +invasion 01#A1,A0 +invention 01#A1,A0 +inventiveness 01#A1,A0 +inventor 01#A1,A0 +inventory 01#A1,A2,A3,A0 +inverse 01#A1,A2,A3 +investigation 01#A1,A2,A0 +investigator 01#A1,A2,A3,A0 +investing 01#A1,A2,A0 +investment 01#A1,A2,A0 +investor 01#A1,A2,A0 +invitation 01#A1,A2,A0 +involvement 01#A1,A2,A0 02#A1,A2 +iota 01#A1,A3 +ire 01#A1,A0 +irony 01#A1,A2,A3,A0 +irradiation 01#A1,A2,A0 +irregularity 01#A1,A2,A0 +irritation 01#A1,A0 +isolation 01#A1,A0 +issuance 01#A1,A2,A3,A0 +issue 01#A2,A0 02#A1,A3 03#A1,A2,A3,A0 04#A1,A0 05#A1 06#A1 07#A1,A2,A0 08#A1,A2,A0 +issuer 01#A1,A2,A3,A0 +issuing 01#A1,A2,A3,A0 +iteration 01#A1 +itinerary 01#A1,A2,A0 +jab 01#A1,A2,A0 +jamming 01#A1,A2,A3,A0 +jaunt 01#A1,A0 +jealousy 01#A1,A2,A0 +jillion 01#A1,A3 +jitters 01#A1,A0 +job 01#A1,A2,A0 +jockey 01#A1,A2,A3,A4,A0 +jockeying 01#A1,A2,A0 +joke 01#A1,A2,A0 +jolt 01#A1,A2,A0 +journey 01#A1,A0 +joy 01#A1,A0 +judge 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +judgment 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +jump 01#A1,A2,A3,A4 02#A1,A0 +jumpiness 01#A0 +juncture 01#A1,A2,A3 +jungle 01#A1,A3 +junior 01#A1,A2,A0 +junket 01#A1,A0 +jurisdiction 01#A1,A0 +jurist 01#A1,A0 +jury 01#A1,A2,A3,A4,A0 +justice 01#A1,A2,A3,A0 02#A2,A0 +justification 01#A1,A2,A3,A4,A0 +keeper 01#A1,A0 +kernel 01#A1,A0 02#A1,A3 +key 01#A1,A2,A0 +keyboard 01#A1 +kicker 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 +kid 01#A1,A0 +kidnapping 01#A1,A0 +killer 01#A1,A2,A0 +killing 01#A1,A2,A0 02#A1,A0 +kilobyte 01#A1,A3 +kilometer 01#A1,A3 +kind 01#A1 02#A1 +king 01#A1,A0 02#A1,A2,A0 +kingpin 01#A1,A2,A0 +kit 01#A1,A0 +kitty 01#A1,A2 +knack 01#A1,A0 +knee 01#A1 +knock 01#A1,A2,A0 +knock-off 01#A1,A2,A0 +knot 01#A1 02#A1,A0 +know-how 01#A1,A2,A0 +knowledge 01#A1,A2,A0 +known 01#A1,A2,A0 +label 01#A1,A2 02#A1,A2,A0 03#A1,A0 +labor 01#A1,A2,A3,A4,A0 +laborer 01#A1,A0 +lack 01#A1,A0 +lady 01#A1,A3,A0 +lag 01#A1,A2,A3 +laggard 01#A1,A0 +lagging 01#A1,A2,A0 +lament 01#A1,A0 +land 01#A1 02#A1 +landing 01#A1,A0 +landlord 01#A1,A0 +landmark 01#A1,A0 02#A1 +landscape 01#A1,A2 +language 01#A1,A2,A0 02#A1,A2,A3 +lapse 01#A1,A0 +latitude 01#A1,A2,A3,A0 +laugh 01#A1,A2,A3,A0 +laughingstock 01#A1,A0 +launch 01#A1,A0 +launching 01#A1,A0 +launderer 01#A1,A0 +laundering 01#A1,A0 +law 01#A1,A2,A3,A0 +lawmaker 01#A1,A2,A0 +lawsuit 01#A1,A2,A3,A4,A0 +lawyer 01#A1,A2,A3,A4,A0 +layer 01#A1,A2,A3 +layoff 01#A1,A0 +lead 01#A1,A2,A3,A4,A0 02#A1,A0 03#A1,A2,A0 04#A1,A2,A3,A0 05#A1,A2,A0 +leader 01#A1,A2,A3,A0 +leadership 01#A1,A0 +leaf 01#A1 +leap 01#A1,A2,A3,A4 02#A1,A0 +lease 01#A1,A2,A3,A4,A0 +leasing 01#A1,A2,A3,A4,A0 +leave 01#A1,A3,A4,A0 +leaving 01#A1,A2,A0 +lecture 01#A1,A2,A0 +lecturer 01#A1,A2,A0 +leeway 01#A1,A0 +leg 01#A1 02#A1,A2,A0 03#A1,A2,A0 +legacy 01#A1,A2,A0 +legality 01#A1,A2,A3 +legalization 01#A1,A0 +legerdemain 01#A1,A2,A0 +legion 01#A1,A2,A3 +legislation 01#A1,A2,A0 +legislator 01#A1,A2,A3,A0 +legislature 01#A1,A0 +legitimacy 01#A1,A2,A3 +lender 01#A1,A2,A0 +lending 01#A1,A2,A0 +length 01#A1,A2 +lens 01#A1 +lesion 01#A1,A2,A0 +lesser 01#A1,A0 +lesson 01#A1,A2,A0 +lethargy 01#A1,A0 +letter 01#A1,A2,A0 02#A1 +level 01#A1,A2,A3 +leveling 01#A1,A2,A0 +leverage 01#A1,A0 +levy 01#A1,A0 +lexicon 01#A1,A0 +liability 01#A1,A2,A3,A0 02#A1,A2,A0 +liaison 01#A1,A2,A0 +libel 01#A1,A0 +liberalization 01#A1,A0 +liberation 01#A1,A2,A0 +librarian 02#A1,A2,A0 +library 01#A1,A2,A3 +license 01#A1,A2,A0 +licensee 01#A1,A2,A0 +licensing 01#A1,A2,A0 +licentiousness 01#A0 +lid 01#A1,A2,A0 +lieutenant 01#A1,A2,A0 02#A1,A2,A0 +life 01#A1,A3,A0 02#A1,A2,A3 03#A1 +lifeblood 01#A1,A0 +lifeline 01#A1,A0 +lifetime 01#A1,A2 +lift 01#A1,A2,A3,A4,AM,A0 +lifting 01#A1,A2,A3,A4,AM,A0 02#A1,A0 +liftoff 01#A1,A2,A3,A4,AM,A0 +light 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2 +like 01#A1,A2 02#A1,A0 +likelihood 01#A1,A2 +likeness 01#A1,A2 +limb 01#A1 +limit 01#A1,A2,A3,A0 +limitation 01#A1,A2,A0 +linage 01#A1,A2,A3 +line 11#A1,A0 01#A1,A3 12#A1 02#A1 13#A1,A2,A0 03#A1,A2,A3 14#A1 04#A1,A2 05#A1,A2,A3,A0 06#A1,A3,A0 07#A1 08#A1 09#A1 10#A1,A0 +line-up 01#A1,A2,A3,A0 +liner 01#A1,A2,A0 +lingo 01#A1,A0 +link 01#A1,A2,A0 +linkage 01#A1,A2,A0 +lip 01#A1 +liquidation 01#A1,A0 +liquidator 01#A1,A0 +liquidity 01#A1,A2,A3 +lira 01#A1,A3 +list 01#A1,A2,A0 02#A1,A2,A0 +listener 01#A1,A0 +listing 01#A1,A2,A0 +litany 01#A1,A3 +literature 01#A1,A3,A0 +litigation 01#A1,A2,A3,A0 +litigator 01#A1,A2,A0 +livelihood 01#A1,A2,A3,A0 +load 01#A1,A2,A3 +loaf 01#A1,A3 +loan 01#A1,A2,A3,A4,A0 +lobby 01#A1,A2,A3 02#A1 +lobbying 01#A1,A2,A0 +lobbyist 01#A1,A2,A0 +location 01#A1,A2,A0 +lock 01#A1,A2,A0 +lockstep 01#A1,A2,A0 +lockup 01#A1,A0 +locution 01#A1,A2,A3,A0 +logic 01#A1,A0 02#A1 +logjam 01#A1,A2,A3,A0 +logo 01#A1,A2,A0 +loitering 01#A0 +longevity 01#A1,A2 +longing 01#A1,A0 +look 01#A1,A2,A0 02#A1,A2 +lookout 01#A1,A2,A0 +loophole 01#A1,A2 +looting 01#A1,A2,A0 +loser 01#A1,A2,A0 02#A1,A2,A0 +loss 01#A1,A2,A3,A0 +lot 01#A1,A3 +lotion 01#A1,A2,A3,A0 +love 01#A1,A0 +lover 01#A1,A0 +low 01#A1,A2 +loyalty 01#A1,A2,A0 +luck 01#A1,A0 +lull 01#A0 +lunch 01#A0 +luncheon +lung 01#A1 +lure 01#A1,A2,A3,A4,A0 +luster 01#A1,A2 +lyric 01#A1,A0 +lyricism 01#A1,A2 +lyricist 01#A1,A2,A0 +machine 01#A1,A0 02#A1,A3 +machinery 01#A1,A2,A3,A0 +magazine 01#A1,A0 +magic 01#A1,A2 +magistrate 01#A1,A0 +magnate 01#A1,A0 +magnetism 01#A1,A2 +magnitude 01#A1,A2 +maid 01#A1,A2,A0 +mail 01#A1,A2,A0 +mailing 01#A1,A2,A0 +mailman 01#A1,A0 +mainstay 01#A1,A0 02#A1 +mainstream 01#A1,A0 02#A1 +maintainence 01#A1,A2,A3,A0 +maintenance 01#A1,A2,A0 +majority 01#A1,A3 +maker 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A2,A0 +makeup 01#A1,A0 +making 01#A1,A2,A3,A0 +malfunction 01#A1 +malignancy 01#A1 +malnourishment 01#A1,A0 +malnutrition 01#A1,A0 +man 01#A1,A2,A3,A0 02#A1,A0 +management 01#A1,A2,A0 +manager 01#A1,A2,A3,A0 +managing 01#A1,A0 +mandate 01#A1,A0 +maneuver 01#A1,A2,A0 +maneuvering 01#A1,A2,A0 +manhood 01#A1,A2 +mania 01#A1,A0 +manifestation 01#A1,A2 +manifesto 01#A1,A2,A3,A0 +manipulation 01#A1,A2,A0 +manipulator 01#A1,A2,A0 +manner 01#A1,A2,A0 02#A1,A0 03#A1 +manpower 01#A1,A0 +manual 01#A1,A2,A0 +manufacture 01#A1,A2,A3,A0 +manufacturer 01#A1,A2,A3,A0 +manufacturing 01#A1,A2,A3,A0 +manuscript 01#A1,A0 +map 01#A1,A3,A0 +march 01#A1,A3,A4 02#A1,A2,A0 +margin 01#A1,A2,A3 +mark 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A2 +markdown 01#A1,A2,A3,A4,A0 +market 01#A1,A2,A3,A4,A0 +marketability 01#A1,A2,A3,A4,A0 +marketer 01#A1,A2,A3,A4,A0 +marketing 01#A1,A2,A3,A4,A0 +marriage 01#A1,A2,A0 +marshal 01#A1,A2,A3,A0 +mass 01#A1,A2 02#A1 +massacre 01#A1,A2,A0 +massage 01#A1,A0 +mastermind 01#A1,A0 +masterminding 01#A1,A0 +masterpiece 01#A1,A0 +match 01#A1,A2,A0 02#A1,A2,A3,A0 +matching 01#A1,A2,A0 +mate 01#A1,A0 02#A1,A2,A0 +matter 01#A1 02#A1 03#A1,A2 04#A1,A3 05#A1,A2 +maturity 01#A1,A2 02#A1 +maxim 01#A1,A2,A3,A0 +maximum 01#A1,A2,A0 +mayor 01#A1,A2,A3,A0 +mayoralty 01#A1,A2,A0 +maze 01#A1,A3 +mean 01#A1,A2,A3,A0 +meaning 01#A1,A0 +measure 01#A1,A2,A0 02#A1,A0 03#A1,A3 +measurement 01#A1,A2,A3,A4,A0 +mechanism 01#A1,A0 +medalist 01#A1,A2,A3,A0 +median 01#A1,A2,A3 +mediation 01#A1,A2,A0 +mediator 01#A1,A2,A3,A0 +medication 01#A1,A2,A3,A0 +medicine 01#A1,A2,A3,A0 +meditation 01#A1,A0 +medium 01#A1,A2 +meeting 01#A1,A2,A8,A9,A0 +megabyte 01#A1,A3 +megawatt 01#A1,A3 +melding 01#A1,A2,A0 +meltdown 01#A1,A0 +member 01#A1 +membership 01#A1,A2,A0 02#A1,A2,A3 +memento 01#A1,A2,A0 +memo 01#A1,A2,A0 +memoir 01#A1,A0 +memorabilia 01#A1,A2,A0 +memorandum 01#A1,A2,A0 +memorial 01#A1,A2,A0 +memory 01#A1,A2,A0 02#A1,A2,A3 03#A1,A2,A0 +menace 01#A1,A2,A0 +mentality 01#A1,A2 +mention 01#A1,A2,A3,A0 +menu 01#A1,A0 +merchant 01#A1,A2,A3,A4,A0 +mercy 01#A1,A0 +merger 01#A1,A2,A3,A0 +merit 01#A1,A0 +mess 01#A1,A3 02#A1,A0 +message 01#A1,A2,A0 +metaphor 01#A1,A2,A0 +meter 01#A1,A3 +method 01#A1,A0 +methodology 01#A1,A0 +mettle 01#A1,A0 +microcosm 01#A1,A2 +middle 01#A1 +middleman 01#A1,A2,A3,A0 +midrange 01#A1 +might 01#A1,A2 +migration 01#A1,A2,A0 +mile 01#A1,A3 02#A1,A2,A3 +milestone 01#A1,A0 +military 01#A1,A2,A3,A0 +militia 01#A1,A2,A3 +mill 01#A1,A2,A3,A0 +million 01#A1,A3 +mincemeat 01#A1,A2,A0 +mind 01#A1,A2,A0 +mine 01#A1,A0 +miner 01#A1,A0 +minimalism 01#A1,A2 +minimum 01#A1,A2,A0 +mining 01#A1,A0 +minister 01#A1,A2,A3,A0 +minority 01#A1,A3 +minute 01#A1,A3 02#A1 03#A1,A2,A0 +miracle 01#A1,A2 +mirror 01#A1,A2 +mirroring 01#A1,A2 +misadventure 01#A1,A0 +miscalculation 01#A1,A2,A0 +mischief 01#A0 +misconception 01#A1,A2,A0 +misdeed 01#A1,A2,A3,A4,A0 +misfortune 01#A1,A0 +misjudgment 01#A1,A2,A3,A0 +mismanagement 01#A1,A0 +mismatch 01#A1,A2,A0 +misperception 01#A1,A0 +misquotation 01#A1,A2,A0 +misrepresentation 01#A1,A2,A0 +mission 01#A1,A0 +misstatement 01#A1,A0 +mistake 01#A1,A2,A0 +mistress 01#A1,A3,A0 +mistrust 01#A1,A0 +misuse 01#A1,A0 +mitigation 01#A1,A0 +mix 01#A1,A2,A3,A0 +mixer 01#A1,A2,A3,A0 +mixture 01#A1,A2,A3,A0 +moat 01#A1 +mob 01#A1,A2,A3 +mobilization 01#A1,A0 +mockery 01#A1,A2,A0 +mode 01#A1,A2,A0 +model 01#A1,A2,A0 02#A1,A0 +modeling 01#A1,A0 +moderation 01#A1,A0 +modernization 01#A1,A0 +modicum 01#A1,A3 +modification 01#A1,A2,A3,A0 +mogul 01#A1,A0 +moisturizer 01#A1,A0 +molecule 01#A1,A3 +moment 01#A1 02#A1,A2,A0 +momentum 01#A1,A2,A3 +moniker 01#A1,A2,A0 +monitor 01#A1,A2,A3,A0 +monitoring 01#A1,A2,A0 +monologue 01#A1,A0 +monopoly 01#A1,A0 +month 01#A1,A2,A3 +mood 01#A1,A2 +moon 01#A1,A3,A0 +moonlighting 01#A1,A0 +mop-up 01#A1,A0 +moral 01#A1,A2 +morale 01#A1,A2 +morality 01#A1,A2 +moratorium 01#A1,A2,A0 +morning 01#A1,A3 02#A1 +mortality 01#A1,A2,A0 +mortgage 01#A1,A2,A3,A0 +mothballing 01#A1,A0 +mother 01#A1,A3,A0 +motivation 01#A1,A2,A0 +motive 01#A1,A0 +motor 01#A1 +mound 01#A1,A3 +mountain 01#A1,A3 +mouth 01#A1 +move 01#A1,A2,A0 02#A1,A0 +movement 01#A1,A2,A0 02#A1,A0 03#A1,A2,A3 +movie 01#A1,A0 +multiple 01#A1,A2,A0 +multitude 01#A1,A3 +municipality 01#A1 +mural 01#A1,A0 +murder 01#A1,A2,A0 +murderer 01#A1,A2,A0 +muscle 01#A1,A2,A3 02#A1 +muse 01#A1,A2,A0 +music 01#A1,A2,A0 +musician 01#A1,A2,A0 +musicianship 01#A1,A2,A0 +muster 01#A1,A2,A0 +mutation 01#A1,A0 +mutiny 01#A1,A0 +mystery 01#A1,A2,A0 +mystique 01#A1,A2 +myth 01#A1,A0 +nagging 01#A1,A2,A0 +name 01#A1,A2 +namesake 01#A1,A0 +naming 01#A1,A2,A0 +narrative 01#A1,A0 +narrator 01#A1,A0 +narrowing 01#A1,A2,A3,A4,A0 +narrowness 01#A1,A2 +nation 01#A1 02#A1 +national 01#A1,A3,A0 +nationalism 01#A1,A2,A3 +nationalist 01#A1,A2,A3 +nationalization 01#A1,A0 +native 01#A1,A0 +nature 01#A1,A2,A3 +navigation 01#A1,A2,A0 +navy 01#A1,A2,A3,A0 +necessity 01#A1,A2,A0 +neck 01#A1,A3 +need 01#A1,A0 +negative 01#A1,A2,A3 +neglect 01#A1,A0 +negligence 01#A1,A0 +negotiating 01#A1,A2,A0 +negotiation 01#A1,A2,A0 +negotiator 01#A1,A2,A0 +neighbor 01#A1,A3,A0 +neighborhood 01#A1,A2,A3 02#A1,A3,A0 03#A1 +nemesis 01#A1,A0 +nephew 01#A1,A3,A0 +nerve 01#A1,A2 02#A1,A0 +nervousness 01#A1,A0 +nest 01#A1,A3,A0 +net 01#A1 02#A1,A2,A3 +network 01#A1,A2,A3 +neurologist 01#A1,A2,A3,A4,A0 +neutralization 01#A1,A0 +newcomer 01#A1,A0 +news 01#A1,A3,A0 +newscast 01#A1,A2,A3,A0 +newsman 01#A1,A2,A3,A0 +niche 01#A1,A3,A0 +nickname 01#A1,A2,A0 +night 01#A1 +nightmare 01#A1,A0 +nobility 01#A1,A2,A3 +nod 01#A1,A0 +noise 01#A1,A2,A3 +nomination 01#A1,A2,A0 +nominee 01#A1,A2,A0 +non-event 01#A1 +norm 01#A1,A2,A0 +normalcy 01#A1,A3 +north 01#A1,A2 +nose 01#A1 +nostalgia 01#A1,A0 +note 01#A1,A2,A0 02#A1,A0 03#A1,A2,A3 +notice 01#A1,A3,A0 02#A1,A2,A0 03#A1,A2,A3,A0 +notification 01#A1,A2,A0 +notion 01#A1,A0 +notoriety 01#A1,A2,A3 +novel 01#A1,A0 +novelist 01#A1,A2,A0 +novice 01#A1,A0 +novitiate 01#A1,A2,A3,A0 +nuance 01#A1,A2,A3 +nuisance 01#A1,A2,A0 +number 01#A1,A3 02#A1,A2,A3 03#A1,A2,A3,A0 +numerator 01#A1 +nutrition 01#A1,A2,A0 +oasis 01#A1,A3 +oath 01#A1,A2,A0 +obedience 01#A1,A0 +obeisance 01#A1,A0 +obfuscation 01#A1,A0 +object 01#A1,A2,A0 02#A1,A0 +objection 01#A1,A2,A0 +objective 01#A1,A0 +objectivity 01#A1,A0 +obligation 01#A1,A2,A0 +oblivion 01#A1,A0 +obscurity 01#A1 +observance 01#A1,A0 +observation 01#A1,A2,A3,A0 +observatory 01#A1 +observer 01#A1,A0 +obsession 01#A1,A0 +obstacle 01#A1,A2,A3,A0 +obstruction 01#A1,A0 +occasion 01#A1,A0 +occupancy 01#A1,A2,A0 +occupant 01#A1,A2,A0 +occupation 01#A1,A2,A0 02#A2,A0 +occurrence 01#A1 +oddity 01#A1,A2 +odds 01#A1,A2,A3 02#A1,A2,A0 +offender 01#A1,A2,A0 +offense 01#A1,A2,A0 +offensive 01#A1,A2,A0 +offer 01#A1,A2,A3,A0 +offering 01#A1,A2,A3,A0 +office 01#A1,A2,A0 02#A1,A2,A3 +officer 01#A1,A2,A3,A0 +official 01#A1,A2,A3,A0 +officialdom 01#A1,A2,A3,A0 +offshoot 01#A1,A0 +offspring 01#A1,A3,A0 +ombudsman 01#A1,A2,A3,A0 +omission 01#A1,A2,A0 +onset 01#A1,A2,A0 +onslaught 01#A1,A3 02#A1,A2,A0 +onus 01#A1,A2,A0 +open 01#A1,A2,A3,A0 02#A1 +opener 01#A1,A2,A3,A0 +opening 01#A1,A2,A3,A0 02#A1 03#A1 04#A1,A0 05#A1,A2,A0 +openness 01#A1,A0 +operating 01#A1,A2,A3,A4,A0 +operation 01#A1,A0 02#A1,A2,A3,A4,A0 +operative 01#A1,A2,A3,A0 +operator 01#A1,A2,A3,A4,A0 +opinion 01#A1,A2,A0 02#A1,A2,A3,A0 +opponent 01#A1,A2,A0 +opportunism 01#A1 +opportunity 01#A1,A0 +opposite 01#A1,A2 +opposition 01#A1,A0 +oppression 01#A1,A0 +optimism 01#A1,A0 +option 01#A1,A2,A3,A9,A0 +orbit 01#A1 +ordeal 01#A1,A0 +order 01#A1,A2,A0 02#A1,A2,A3,A0 03#A1,A2,A0 04#A1,A2 05#A1,A2,A0 06#A1,A2 +ordinance 01#A1,A0 +organ 01#A1 +organization 01#A1,A2,A3 +organizer 01#A1,A2,A3,A4,A0 +organizing 01#A1,A2,A3,A4,A0 +origin 01#A1,A2,A4,A0 +origination 01#A1,A2,A4,A0 +originator 01#A1,A2,A4,A0 +other 01#A1,A3 +ounce 01#A1,A3 +ouster 01#A1,A2,A0 +outage 01#A1,A2,A3,A0 +outbreak 01#A1,A2 +outcome 01#A1,A2 +outcry 01#A1,A2,A0 +outfielder 01#A1,A2,A3,A0 +outfit 01#A1,A0 +outflow 01#A1,A2,A3 +outgrowth 01#A1,A2 +outlay 01#A1,A2,A3,A4,A0 +outlet 01#A1,A0 +outline 01#A1,A0 +outlook 01#A1,A0 +outpost 01#A1,A2 02#A1 +output 01#A1,A0 +outrage 01#A1,A0 +outset 01#A1 +outside 01#A1 +outskirts 01#A1 +ovation 01#A1,A2,A0 +overbreadth 01#A1,A2,A3 +overbuilding 01#A0 +overcharge 01#A1,A2,A3,A0 +overdependence 01#A1,A0 +overhang 01#A1,A2,A3,A4,A0 +overhaul 01#A1,A2,A3,A0 +overhead 01#A1,A2,A3 +overlay 01#A1,A2,A0 +overproduction 01#A1,A0 +overreaction 01#A1,A2,A0 +overrun 01#A1,A0 +overseer 01#A1,A0 +oversight 01#A1,A0 02#A0 +overstatement 01#A1,A2,A0 +oversupply 01#A1,A2,A3,A0 +overture 01#A1,A2,A0 +overuse 01#A1,A0 +owner 01#A1,A2,A0 +ownership 01#A1,A2,A0 +pace 01#A1,A2,A3,A0 +pacing 01#A1,A0 +pack 01#A1,A2,A3 +package 01#A1,A2,A0 +packaging 01#A1,A2,A3,A0 +packet 01#A1,A3 +pact 01#A1,A2,A0 +paean 01#A1,A0 +page 01#A1,A2 02#A1,A2,A0 +pageant 01#A1,A3 +pain 01#A1,A2,A0 +painting 01#A1,A2,A0 02#A1,A2,A0 +pair 01#A1,A3 +pal 01#A1,A3,A0 +palate 01#A1,A0 +pallor 01#A1,A2,A3 +palm 01#A1 +pamphleteer 01#A1,A2,A3,A0 +panel 01#A1,A2,A3 02#A1 +pang 01#A1,A2,A0 +panic 01#A1,A0 +panjandrum 01#A1,A0 +paper 01#A1,A0 02#A1,A2,A3,A0 03#A1,A2,A3,A0 +paperwork 01#A1,A2,A3 +parade 01#A1,A0 02#A1,A3 +paradise 01#A1 +paradox 01#A1,A2,A0 +paragraph 01#A1,A2,A0 +parallel 01#A1,A0 +paralysis 01#A1,A0 +parameter 01#A1,A2,A3 +paraphernalia 01#A1,A0 +pardon 01#A1,A2,A0 +parent 01#A1,A3,A0 +parishioner 01#A1 +parity 01#A1,A2,A3 +parking 01#A1,A2,A0 +parlance 01#A1,A2,A3 +parliament 01#A1,A2,A3 +parody 01#A1,A0 +part 01#A1 02#A1,A2,A3,A0 03#A1,A0 04#A1,A2,A0 05#A1 +participant 01#A1,A0 +participation 01#A1,A0 +particular 01#A1 02#A1,A2 +partner 01#A1,A2,A0 +partnership 01#A1,A2,A0 +party 01#A1,A2,A0 02#A1,A2,A3 03#A1,A0 +pass 01#A1,A0 02#A1,A0 03#A1,A2,A0 +passage 01#A1,A2,A0 02#A1,A2 +passel 01#A1,A3 +passenger 01#A1,A0 +passing 01#A1 +passion 01#A1,A0 +passivity 01#A0 +password 01#A1,A2,A0 +past 01#A1,A3 +pastime 01#A1,A0 +pastor 01#A1,A2,A3,A0 +pasture 01#A1 +patch 01#A1,A3 +patchwork 01#A1,A2,A3,A0 +patent 01#A1,A0 +path 01#A1,A0 +patience 01#A1,A0 +patient 01#A1,A2,A0 +patriarch 01#A1,A2,A3,A0 +patriotism 01#A1,A2,A3 +patrol 01#A1,A0 +patrolling 01#A1,A0 +patron 01#A1,A0 +pattern 01#A1,A2,A3 +pause 01#A0 02#A1,A2,A0 +pay 01#A1,A2,A3,A0 +paycheck 01#A1,A2,A3,A0 +payer 01#A1,A2,A3,A0 +payment 01#A1,A2,A3,A0 +payout 01#A1,A2,A3,A0 +payroll 01#A1,A2,A3,A0 +peace 01#A1,A0 +peak 01#A1,A2,A3 +peal 01#A1,A0 +peck 01#A1,A3 +peculiarity 01#A1,A2 +peddler 01#A1,A2,A3,A0 +peek 01#A1,A0 +peer 01#A1,A2 +penalty 01#A1,A2,A3,A0 +penchant 01#A1,A0 +pennant 01#A1,A2,A0 +pension 01#A1,A2,A3,A0 +perc-sign 01#A1,A3 +percent 01#A1,A3 +percentage 01#A1,A3 +perception 01#A1,A2,A0 02#A1,A0 +perceptiveness 01#A1,A2,A0 02#A1,A2 +perfection 01#A1,A0 +performance 01#A1,A2,A0 02#A1,A2,A0 +performer 01#A1,A2,A0 +peril 01#A1,A0 +perimeter 01#A1,A2,A3 +period 01#A1 +periphery 01#A1 +perjury 01#A1,A2,A0 +perk 01#A1,A0 +permanence 01#A1,A2,A3 +permission 01#A1,A2,A0 +permit 01#A1,A2,A0 +persecution 01#A1,A0 +persistence 01#A1,A0 +person 01#A1,A2,A3,A0 02#A1,A3,A0 +personality 01#A1,A2 +personification 01#A1,A2 +personnel 01#A1,A2,A0 +perspective 01#A1,A2,A0 +persuasion 01#A1,A2,A0 +persuasiveness 01#A1,A2 +perversion 01#A1,A2,A3,A0 +perversity 01#A1 +peseta 01#A1,A3 +pessimism 01#A1,A0 +pet 01#A1,A3,A0 +petition 01#A1,A2,A0 +pettiness 01#A1,A2 +phalanx 01#A1,A3 +pharaoh 01#A1,A2,A3,A0 +phase 01#A1 +phase-out 01#A1,A0 +philosophy 01#A1,A2,A0 +phobia 01#A1,A0 +photo 01#A1,A3,A0 +photograph 01#A1,A3,A0 +phrase 01#A1,A0 +physicist 01#A1,A2,A3,A0 +pianist 01#A1,A2,A0 +pick 01#A1,A2,A3,A0 +picker 01#A1,A2,A3,A0 +pickup 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A4 +picture 01#A1,A0 +piece 01#A1 02#A1 03#A1,A3,A0 04#A1,A2,A0 +pile 01#A1,A3 +pileup 01#A1,A0 +pilgrimage 01#A1,A0 +pill 01#A1,A2,A3,A0 +pillar 01#A1,A0 +pilot 01#A1,A2,A3,A0 02#A1,A3,A0 03#A1 +pinning 01#A1,A2,A0 +pint 01#A1,A3 +pioneer 01#A1,A0 +pipeline 01#A1,A2,A3,A0 +piracy 01#A1,A2,A0 +pirate 01#A1,A2,A0 +pistil 01#A1 +pitch 01#A1,A2,A0 02#A1,A2,A3,A0 +pitfall 01#A1,A2,A3,A0 +place 01#A1,A2,A3 02#A1,A0 03#A1 04#A1,A2,A3,A0 05#A1,A2,A3,A0 06#A1 07#A1,A2 08#A1 09#A1,A2,A3 +placement 01#A1,A2,A0 +placing 01#A1,A2,A0 +plan 01#A1,A2,A3,A0 +plank 01#A1,A0 +planner 01#A1,A2,A3,A0 +planning 01#A1,A2,A3,A0 +plantation 01#A1,A2,A0 +planting 01#A1,A2,A0 +plateau 01#A1 +platform 01#A1,A2,A3,A0 02#A1,A0 +platitude 01#A1,A0 +platoon 01#A1,A2,A3 +play 01#A1,A0 02#A1,A2,A0 03#A1,A0 04#A1,A2,A0 05#A1,A2 06#A1,A2,A0 07#A1,A2,A0 +player 01#A1,A2,A0 02#A1,A0 +playing 01#A1,A2,A0 02#A1,A0 +playoff 01#A1,A2,A0 +plea 01#A1,A2,A0 02#A1,A2,A0 +pleasure 01#A1,A2,A0 +pledge 01#A1,A2,A0 +plenty 01#A1,A3 +plethora 01#A1,A3 +plight 01#A1,A0 +plot 01#A1,A2,A0 02#A1,A0 03#A1,A2 +plotter 01#A1,A0 +ploy 01#A1,A2,A3,A0 +plug 01#A1,A0 02#A1,A0 +plume 01#A1,A3 +plummet 01#A1,A2,A3,A4,AM +plunge 01#A1,A2,A3,A4,AM 02#A1,A2,A0 +plunging 01#A1,A2,A3,A4,AM +plurality 01#A1,A3 +point 01#A1,A2 02#A1,A2,A0 03#A1,A2,A3 04#A1,A3 05#A1 +pointer 01#A1,A2,A0 +police 01#A1,A2,A3,A0 +policewoman 01#A1,A2,A3,A0 +policy 01#A1,A2,A0 +policyholder 01#A1,A2,A3,A0 +polish 01#A1,A2,A3 +polishing 01#A1,A2,A3,A0 +politics 01#A1,A2,A3 02#A1 +poll 01#A1,A2,A0 +pollination 01#A1,A0 +polling 01#A1,A2,A0 +pollutant 01#A1,A0 +pollution 01#A1,A0 +pomologist 01#A1,A2,A3,A0 +pool 01#A1,A0 +populace 01#A1,A2,A3 +popularity 01#A1,A2,A3 +population 01#A1,A2,A3 +populism 01#A1,A2,A3 +port 01#A1 +portfolio 01#A1,A2,A0 +portion 01#A1,A2,A0 +portrait 01#A1,A0 +portrayal 01#A1,A2,A0 +position 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2,A0 +positioning 01#A1,A2,A0 +possession 01#A1,A0 +possibility 01#A1,A2 +post 01#A1,A2,A0 +posting 01#A1,A2,A0 +postmaster 01#A1,A2,A3,A0 +postponement 01#A1,A2,A3,A4,A0 +posture 01#A1,A0 +pot 01#A1,A3 +potentate 01#A1,A2,A3,A0 +potential 01#A1,A0 +potentiality 01#A1,A2 +potpourri 01#A1,A3 +pound 01#A1,A3 +poverty 01#A1,A2 +power 01#A1,A2,A3,A0 +powerhouse 01#A1,A2,A0 +powwow 01#A1,A0 +practice 01#A1,A2,A0 +practicing 01#A1,A2,A0 +practitioner 01#A1,A2,A0 +pragmatism 01#A1,A2 +praise 01#A1,A2,A0 +prattle 01#A1,A2,A0 +prayer 01#A1,A2,A0 +pre-eminence 01#A3,A0 +preamble 01#A1 +precaution 01#A1,A2,A0 +precedent 01#A1,A2,A0 +precision 01#A1,A2,A3 +preclearance 01#A1,A0 +precondition 01#A1,A2,A0 +precursor 01#A1,AA,A0 +predator 01#A1,A2,A0 +predecessor 01#A1,AA,A0 +predicament 01#A1,A2,A3,A0 +predictability 01#A1,A2 +prediction 01#A1,A0 +predilection 01#A1,A2,A0 +preference 01#A1,A2,A0 +pregnancy 01#A0 +prejudice 01#A1,A2,A0 +prelude 01#A1,AA,A0 02#A1,A0 +premier 01#A1,A2,A3,A0 +premiere 01#A1,A0 +premise 01#A1,A2,A0 +premium 01#A1,A2,A3 +preoccupation 01#A1,A0 +preparation 01#A1,A2,A3,A0 02#A1,A2,A0 +preparer 01#A1,A2,A3,A0 +prepayment 01#A1,A2,A3,A0 +prerequisite 01#A1,A2,A0 +prerogative 01#A1,A0 +prescription 01#A1,A0 02#A1,A2,A0 +presence 01#A1,A2 +presentation 01#A1,A2,A3,A0 +presenter 01#A1,A2,A3,A0 +preservation 01#A1,A2,A0 +presidency 01#A1,A2,A0 +president 01#A1,A2,A3,A0 +press 01#A1,A2,A3 02#A1,A2,A3 03#A1,A2,A0 04#A1,A2,A0 +pressure 01#A1,A2,A0 02#A1,A2 +prestige 01#A1,A2,A3 +presumption 01#A1,A2,A0 +pretense 01#A1,A0 +pretension 01#A1,A0 +pretext 01#A1,A2,A0 +prevalance 01#A1,A2 +prevention 01#A1,A2,A3,A0 +preview 01#A1,A0 +prey 01#A1,A0 +price 01#A1,A2,A3,A0 +pricing 01#A1,A2,A0 +pride 01#A1,A0 +priest 01#A1,A2,A3,A4,A0 +primary 01#A1,A2,A0 +prince 01#A1,A2,A3,A0 +principal 01#A1,A2,A3,A0 02#A1,A0 03#A1,A2,A3 +principle 01#A1,A2,A3,A0 02#A1 03#A1,A2,A3 +printer 01#A1,A2,A0 +printing 01#A1,A2,A0 +priority 01#A1,A2,A3,A4,A0 +prisoner 01#A1,A2,A3,A0 +privacy 01#A1,A2,A3,A0 +privatization 01#A1,A0 +privilege 01#A1,A0 +prize 01#A1,A2,A3,A0 +pro 01#A1,A2,A0 +probability 01#A1,A2 +probation 01#A1,A2,A0 +probe 01#A1,A2,A0 +problem 01#A1,A2,A3,A0 02#A1,A0 +problematic 01#A1,A2,A3,A0 +procedure 01#A1,A0 +proceeding 01#A1,A2,A3,A0 +proceeds 01#A1,A2,A3,A0 +process 01#A1,A0 +processing 01#A1,A2,A3,A0 +processor 01#A1,A2,A3,A0 +proclamation 01#A1,A2,A0 +procrastination 01#A0 +procreation 01#A1,A0 +procurement 01#A1,A2,A0 +prodding 01#A1,A2,A0 +prodigy 01#A1,A0 +producer 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +product 01#A1,A2,A3,A0 +production 01#A1,A2,A3,A0 +productivity 01#A1,A2,A3 +profession 01#A1,A2,A0 02#A1,A2,A3,A0 +professional 01#A1,A2,A0 +professionalism 01#A1,A2,A3 +professor 01#A1,A2,A3,A4,A0 +proficiency 01#A1,A0 +profile 01#A1,A3,A0 +profit 01#A1,A2,A3,A0 +profitability 01#A1,A2,A0 +progenitor 01#A1,A3,A0 +program 01#A1,A0 02#A1,A0 +programmer 01#A1,A2,A0 +programming 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +progress 01#A1,A3,A4 +progression 01#A1,A3,A4 +prohibition 01#A1,A2,A3,A0 +project 01#A1,A0 +projection 01#A1,A3,A0 +proliferation 01#A0 +prominence 01#A1,A2 +promise 01#A1,A2,A0 +promoter 01#A1,A2,A0 02#A1,A0 +promotion 01#A1,A2,A3,A0 02#A1,A2,A0 +pronunciation 01#A1,A2,A0 +proof 01#A1,A2,A0 +propaganda 01#A1,A0 +propensity 01#A1,A0 +property 01#A1,A0 02#A1,A2,A3 +proponent 01#A1,A2,A0 +proportion 01#A1,A2,A3 +proposal 01#A1,A2,A3,A0 +proposition 01#A1,A2,A3,A0 +proprieter 01#A1,A0 +proprietor 01#A1,A0 +proprietorship 01#A1,A0 +propriety 01#A1,A2,A3 +prose 01#A1,A0 +prosecution 01#A1,A2,A0 +prosecutor 01#A1,A2,A0 +prospect 01#A1,A2,A0 +prospectus 01#A1,A3,A0 +prosperity 01#A1,A2,A0 +prostitution 01#A0 +protagonist 01#A1,A3,A0 +protection 01#A1,A2,A3,A0 +protectionism 01#A1,A2,A3,A0 +protector 01#A1,A2,A3,A0 +protege 01#A1,A3,A0 +protein 01#A1 +protest 01#A1,A2,A0 +protester 01#A1,A0 +protocol 01#A1,A2,A3,A0 +provider 01#A1,A2,A0 +province 01#A1,A2 02#A1,A0 +provision 01#A1,A2,A0 +proviso 01#A1,A2,A0 +provocation 01#A1,A2,A0 +provost 01#A1,A2,A3,A0 +prowess 01#A1,A0 +prowl 01#A1,A0 +proxy 01#A1,A0 +psyche 01#A1 +psychology 01#A1,A2,A3 +public 01#A1,A2,A3,A0 02#A1 +publication 01#A1,A2,A0 +publicist 01#A1,A2,A0 +publicity 01#A1,A2 +publisher 01#A1,A2,A3,A4,A0 +publishing 01#A1,A0 +pull 01#A1,A2 +pullback 01#A1,A2,A3,A4,A0 +pullout 01#A1,A2,A3,A4,A0 +pulse 01#A1,A2,A3 +pummeling 01#A1,A2,A0 +pump 01#A1,A2,A0 +punching 01#A1,A0 +pundit 01#A1,A0 +punishment 01#A1,A2,A0 +purchase 01#A1,A2,A3,A4,A0 +purchaser 01#A1,A2,A3,A4,A0 +purchasing 01#A1,A2,A3,A4,A0 +purification 01#A1,A2,A0 +purist 01#A1,A0 +purity 01#A1,A2,A3 +purpose 01#A1,A2,A3,A4,A0 +pursuer 01#A1,A0 +pursuing 01#A1,A0 +pursuit 01#A1,A0 02#A1,A0 +push 01#A1,A2,A0 02#A1,A2,A0 +pusher 01#A1,A2,A0 +pusillanimity 01#A1,A2 +puzzlement 01#A1,A2,A0 +quadrupling 01#A1,A2,A3,A4,A0 +quake 01#A1 +qualification 01#A1,A2 +quality 01#A1,A2,A3 +quantification 01#A1,A0 +quantity 01#A1,A2,A3 +quarry 01#A1,A0 02#A1,A0 +quarter 01#A1 02#A1,A2,A3 +quarterback 01#A1,A2,A3,A0 +queen 01#A1,A2,A3,A0 +query 01#A1,A2,A0 +quest 01#A1,A0 02#A1,A0 +question 01#A1,A2,A0 02#A1,A2 +quip 01#A1,A2,A0 +quirk 01#A1,A2 +quota 01#A1,A2,A0 +quotation 01#A1,A2,A0 +quote 01#A1,A2,A0 +race 01#A1,A2,A3 02#A1,A2,A3 +rack 01#A1,A2,A3,A0 +racket 01#A1,A0 +radiation 01#A1,A2,A0 +radioactivity 01#A1,A2 +raft 01#A1,A3 +rage 01#A1,A0 02#A1,A2 +raid 01#A1,A0 +raider 01#A1,A0 +railing 01#A1 +rain 01#A1,A2,A0 +rainbow 01#A1,A3 +raise 01#A1,A2,A3,A0 +raiser 01#A1,A2,A0 +raising 01#A1,A2,A0 +rally 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +ramification 01#A1,A0 +ramp 01#A1 +rampage 01#A1,A0 +rancor 01#A1,A0 +range 01#A1,A2,A3,A4,A5 +rank 01#A1,A2,A3 02#A1 03#A1,A2,A0 +ranking 01#A1,A2,A0 +ransom 01#A1,A2,A0 +rap 01#A1,A2,A0 02#A1,A2,A3,A0 +rape 01#A1,A0 +rapidity 01#A1,A2 +rapport 01#A1,A2,A0 +rapprochement 01#A1,A2,A0 +rash 01#A1,A3 +rate 01#A1,A2,A3 +ratification 01#A1,A2,A0 +rating 01#A1,A2,A3,A0 +ratio 01#A1,A2,A3,A0 +rationale 01#A1,A2,A0 +rationality 01#A1,A2 +rationalization 01#A1,A2,A0 +ravage 01#A1,A2,A0 +reach 01#A1,A0 +reaching 01#A1,A0 +reaction 01#A1,A2,A0 +reader 01#A1,A2,A3,A0 +readership 01#A1,A2,A3,A0 +readiness 01#A1,A0 +reading 01#A1,A2,A3,A0 +readjustment 01#A1,A2,A3,A4,A0 +realignment 01#A1,A2,A0 +realism 01#A1,A2,A0 +reality 01#A1,A2 +realization 01#A1,A0 02#A1,A2,A0 +realm 01#A1 +ream 01#A1,A3 +reappearance 01#A1 +reappraisal 01#A1,A2,A0 +rearing 01#A1,A0 +rearrangement 01#A1,A0 +reason 01#A1 02#A1,A2,A3,A4,A0 +reasoning 01#A1,A0 +reassessment 01#A1,A0 +reassignment 01#A1,A2,A0 +reassurance 01#A1,A2,A0 +reauthorization 01#A1,A2,A3,A0 +rebate 01#A1,A2,A3,A4,A0 +rebellion 01#A1,A0 +rebirth 01#A1 +rebound 01#A1,A2,A3,A4 +rebuff 01#A1,A0 +recalculation 01#A1,A2,A0 +recall 01#A1,A2,A0 02#A1,A2,A3,A0 +recantation 01#A1,A0 +recapitalization 01#A1,A2,A0 +receipt 01#A1,A2,A3,A4,A0 02#A1,A2,A0 +receivable 01#A1,A2,A3,A4,A0 +receiver 01#A1,A2,A3,A4,A0 02#A1,A0 +receivership 01#A1,A2,A3,A0 +reception 01#A1,A2,A0 02#A1,A2,A3,A0 +receptivity 01#A1,A0 +receptor 01#A1,A2,A3,A4,A0 +recession 01#A1,A0 +recipe 01#A1,A0 +recipient 01#A1,A2,A3,A4,A0 +recital 01#A1,A0 +reckoning 01#A1,A2,A0 +recognition 01#A1,A2,A0 02#A1,A0 +recombination 01#A1,A2,A3,A0 +recommendation 01#A1,A2,A3,A0 +reconciliation 01#A1,A2,A0 +reconsideration 01#A1,A0 +record 01#A1,A2,A0 02#A1,A2,A3 03#A1,A2,A3 04#A1,A2,A0 05#A1,A0 +recorder 01#A1,A0 +recording 01#A1,A0 +recourse 01#A1,A2,A0 +recovery 01#A1,A2 02#A1,A2,A0 +recruit 01#A1,A2,A3,A0 +recruiter 01#A1,A2,A3,A4,A0 +recruiting 01#A1,A2,A3,A0 +recruitment 01#A1,A2,A3,A0 +rectification 01#A1,A0 +recuperation 01#A1,A2 +recurrence 01#A1,A2 +recycling 01#A1,A2,A0 +redefinition 01#A1,A2,A0 +redemption 01#A1,A2,A0 +redeployment 01#A1,A2,A0 +redevelopment 01#A1,A2,A3,A4,A0 +redistribution 01#A1,A2,A3,A0 +redistributionism 01#A1,A2,A3,A0 +redoubt 01#A1,A0 +reduction 01#A1,A2,A3,A4,AM,A0 +reference 01#A1,A2,A0 +referendum 01#A1,A2,A0 +referral 01#A1,A2,A0 +refinancing 01#A1,A2,A0 +refiner 01#A1,A2,A0 +refinery 01#A1,A2,A0 +refining 01#A1,A2,A0 +reflection 01#A1,A2 02#A1,A0 +refocusing 01#A1,A2,A0 +reform 01#A1,A2,A3,A0 +reformer 01#A1,A2,A3,A0 +reformulation 01#A1,A0 +refuge 01#A1,A2,A0 +refugee 01#A1,A2 +refund 01#A1,A2,A3,A0 +refunding 01#A1,A2,A3,A0 +refurbishing 01#A1,A0 +refurbishment 01#A1,A0 +refusal 01#A1,A2,A0 +refuse 01#A1,A2,A0 +regard 01#A1,A2,A0 02#A1 +regime 01#A1,A0 +regimentation 01#A1,A2,A0 +region 01#A1,A2 02#A1 +register 01#A1,A2,A3 +registration 01#A1,A2,A3,A0 +regret 01#A1,A0 +regularity 01#A1,A2,A3,A0 +regulation 01#A1,A2,A0 +regulator 01#A1,A2,A0 +rehabilitation 01#A1,A2,A0 +rehash 01#A1,A0 +rehearing 01#A1,A2,A0 +reign 01#A1,A0 +reimbursement 01#A1,A2,A3,A0 +rein 01#A1,A0 +reinforcement 01#A1,A2,A0 +reinstatement 01#A1,A2,A0 +reinsurance 01#A1,A2,A3,A4,A0 +reinterpretation 01#A1,A2,A0 +reinvestment 01#A1,A2,A0 +rejection 01#A1,A3,A0 +rejuvenation 01#A1,A0 +relation 01#A1,A2,A0 02#A1,A2,A0 +relationship 01#A1,A2,A0 +relative 01#A1,A3,A0 +relaxation 01#A1,A0 +release 01#A1,A2,A3,A0 +relevance 01#A1,A2,A3 +relevancy 01#A1,A2,A3 +reliance 01#A1,A2,A0 +relief 01#A1,A2,A0 02#A1,A2,A0 +religion 01#A1,A2,A3 +relish 01#A1,A0 +relocation 01#A1,A2,A3,A0 +reluctance 01#A1,A0 +remain 01#A1,A2,A3 +remainder 01#A1,A2 +remark 01#A1,A2,A3,A0 +remarketing 01#A1,A2,A3,A4,A0 +rematch 01#A1,A2,A0 +remedy 01#A1,A2,A0 +remembrance 01#A1,A2,A0 +reminder 01#A1,A2,A3,A0 +remittance 01#A1,A2,A3,A0 +remnant 01#A1 +remodeling 01#A1,A0 +removal 01#A1,A2,A0 +renaissance 01#A1,A2 +rendering 01#A1,A2,A0 +rendezvous 01#A1,A0 +rendition 01#A1,A0 +renegotiation 01#A1,A2,A0 +renewal 01#A1,A0 +renovation 01#A1,A0 +renown 01#A1,A2,A0 +rent 01#A1,A2,A3,A4,A0 +rental 01#A1,A2,A3,A4,A0 +renter 01#A1,A2,A3,A4,A0 +renunciation 01#A1,A2,A0 +reordering 01#A1,A0 +reorganization 01#A1,A2,A3,A4,A0 +repair 01#A1,A0 +reparation 01#A1,A0 +repatriation 01#A1,A2,A0 +repayment 01#A1,A2,A3,A0 +repeal 01#A1,A0 +repeat 01#A1,A2 +repeater 01#A1,A2,A3,A4,A0 +repercussion 01#A1,A0 +repertoire 01#A1,A2,A0 +repetition 01#A1,A2 +replacement 01#A1,A2,A0 +replay 01#A1,A2 +replica 01#A1,A0 +reply 01#A1,A2,A0 +report 01#A1,A2,A0 +reporter 01#A1,A2,A3,A0 +reporting 01#A1,A2,A0 +repository 01#A1,A2,A0 +representation 01#A1,A0 +representative 01#A1,A2,A4,A0 +repression 01#A1,A0 +reprieve 01#A1,A2,A0 +reproduction 01#A1,A0 +reproval 01#A1,A2,A0 +repudiation 01#A1,A0 +repurchase 01#A1,A2,A0 +reputation 01#A1,A2,A0 +request 01#A1,A2,A3,A0 +requirement 01#A1,A2,A3,A0 +requisition 01#A1,A0 +rerun 01#A1 +resale 01#A1,A2,A3,A4,A0 +rescue 01#A1,A2,A0 +research 01#A1,A0 +researcher 01#A1,A0 +resemblance 01#A1,A2 +resentment 01#A1,A2,A0 +reservation 01#A1,A2,A3,A0 +reserve 01#A1,A2,A0 02#A1,A0 +reservoir 01#A1,A3 +reshuffle 01#A1,A0 +reshuffling 01#A1,A0 +residence 01#A1,A0 +resident 01#A1,A0 +residue 01#A1,A3 +resignation 01#A1,A0 +resilience 01#A1,A2 +resiliency 01#A1,A2,A3 +resistance 01#A1,A0 +resolution 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +resolve 01#A1,A2,A3,A0 +resonance 01#A1,A2,A3 +resource 01#A1,A2,A0 +respect 01#A1,A2,A0 02#A1 +respectability 01#A1,A2,A0 +respondent 01#A1,A2,A0 +response 01#A1,A2,A0 +responsibility 01#A1,A2,A3,A0 +rest 01#A1,A3 +restarter 01#A1,A0 +restatement 01#A1,A2,A3,A0 +restitution 01#A1,A2,A0 +restoration 01#A1,A2,A0 02#A1,A2,A0 +restorer 01#A1,A2,A0 +restraint 01#A1,A2,A3,A0 +restriction 01#A1,A2,A3,A0 +restructuring 01#A1,A2,A3,A0 +result 01#A1,A2 +resumption 01#A1,A2,A0 +resurgence 01#A1,A2 +resurrection 01#A1,A0 +retailer 01#A1,A2,A3,A4,A0 +retailing 01#A1,A2,A3,A4,A0 +retainer 01#A1,A2,A3,A0 +retaliation 01#A1,A0 +retardant 01#A1,A0 +retardation 01#A1,A0 +retention 01#A1,A3,A0 +rethink 01#A1,A0 +reticence 01#A1,A0 +retiree 01#A1,A0 +retirement 01#A1,A0 02#A1 +retort 01#A1,A2,A0 +retreat 01#A1,AA,A0 +retrenchment 01#A0 +retribution 01#A1,A0 +retrieval 01#A1,A2,A0 +return 01#A1,A2,A3,A4,AM 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2,A0 05#A1,A2,A3,A0 +reunification 01#A1,A2,A3,A0 +reunion 01#A1,A2,A3,A0 +revamping 01#A1,A0 +revel 01#A1,A0 +revelation 01#A1,A2,A3,A0 +revenge 01#A1,A2,A3,A0 +revenue 01#A1,A2,A3,A0 +reverberation 01#A1 +reverence 01#A1,A2,A0 +reversal 01#A1,A0 +review 01#A1,A2,A3,A0 02#A1,A2,A0 +revision 01#A1,A2,A3,A0 +revival 01#A1,A0 +revolution 01#A1,A2,A3,A0 02#A1,A0 +reward 01#A1,A2,A0 +rhetoric 01#A1,A0 +rhythm 01#A1,A2,A3 +richness 01#A1,A2,A3 +ride 01#A1,A0 +rider 01#A1,A0 02#A1,A2,A0 +ridership 01#A1,A0 +ridicule 01#A1,A2,A0 +rift 01#A1,A2,A0 +right 01#A1,A3,A0 +rigidity 01#A1,A0 +rigor 01#A1,A2,A3 +ring 01#A1,A2,A3 +ringer 01#A1,A2,A0 02#A1,A2 +riot 01#A1,A2,A0 02#A1,A2,A3,A0 +ripple 01#A1,A0 +rise 01#A1,A2,A3,A4,AM +risk 01#A1,A0 +riskiness 01#A1,A2,A3 +rite 01#A1,A2,A0 +ritual 01#A1,A2,A0 +rival 01#A1,A2,A0 +rivalry 01#A1,A2,A0 +road 01#A1,A0 +roadblock 01#A1,A2,A3,A0 +roar 01#A1,A2,A0 +robber 01#A1,A0 +robbery 01#A1,A0 +role 01#A1,A2,A0 +roll 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A3 04#A1,A0 +roll-out 01#A1,A0 +roller 01#A1,A0 +romance 01#A1,A0 +roof 01#A1 +room 01#A1,A2,A3,A0 02#A1 +roomful 01#A1,A3 +roommate 01#A1,A3,A0 +root 01#A1,A2,A4,A0 +roster 01#A1,A2,A3 +round 01#A1,A3 +rout 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +route 01#A1,A0 +routine 01#A1,A0 +row 01#A1,A3 +royalty 01#A1,A2,A3,A0 +rubric 01#A1 +rule 01#A1,A2,A3,A0 02#A1,A0 +ruler 01#A1,A0 +ruling 01#A1,A2,A3,A0 +rumbling 01#A1,A0 +rumor 01#A1,A0 +run 01#A1,A3 02#A1,A2,A3,A4,A0 03#A0 04#A1 05#A1,A0 +run-in 01#A1,A2,A0 +run-up 01#A1,A0 +rundown 01#A1,A0 +runner 01#A1,A2,A3,A0 +runway 01#A1 +rush 01#A1,A2,A0 02#A1,A3 +sabotage 01#A1,A2,A0 +sack 01#A1,A0 02#A1,A3 +sacking 01#A1,A0 +sacrifice 01#A1,A2,A0 +safeguard 01#A1,A2,A0 +safekeeping 01#A1,A2,A3,A0 +safety 01#A1,A2,A3 +saga 01#A1,A0 +sage 01#A1,A2,A0 +sailing 01#A1,A0 +salary 01#A1,A2,A3,A0 +sale 01#A1,A2,A3,A4,A0 +salesman 01#A1,A2,A3,A4,A0 +salvage 01#A1,A2,A0 +salvation 01#A1,A2,A0 +salvo 01#A1,A0 +sample 01#A1,A3,A0 +sanction 01#A1,A2,A0 +sanctity 01#A1 +sandwich 01#A1,A2,A0 +satire 01#A1,A2 02#A1,A0 +satisfaction 01#A1,A2,A0 +saver 01#A1,A2,A3,A0 +saving 01#A1,A2,A3,A0 +savior 01#A1,A2,A0 +say 01#A1,A0 +saying 01#A1,A2,A3,A0 +scale 01#A1,A2,A3 +scaling 01#A1,A2,A3,A4,A5,A0 +scalp 01#A1 +scam 01#A1,A0 +scandal 01#A1,A0 +scapegoat 01#A1,A2,A0 +scapegoating 01#A1,A2,A0 +scarcity 01#A1,A2 +scavenger 01#A1,A2,A0 +scenario 01#A1,A0 +scene 01#A1,A2,A3 02#A1,A2,A0 03#A1 +scent 01#A1,A2 +schedule 01#A1,A2,A3,A0 +scheduling 01#A1,A2,A3,A0 +scheme 01#A1,A2,A3,A0 +scholar 01#A1,A0 +scholarship 01#A1,A0 +school 01#A1 +schoolmate 01#A1,A3,A0 +science 01#A1 +scientist 01#A1,A2,A3,A0 +scoop 01#A1,A0 +scope 01#A1,A2,A3 +score 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A3,A0 04#A1,A3 +scorecard 01#A1,A2,A0 +scorn 01#A1,A2,A0 +scourge 01#A1,A2,A0 +scramble 01#A1,A0 +scrape 01#A1,A0 +screen 01#A1 +screening 01#A1,A2,A0 02#A1,A0 +screenplay 01#A1,A0 +scribe 01#A1,A2,A3,A0 +script 01#A1,A0 +scruff 01#A1 +scrutiny 01#A1,A2,A0 +sea 01#A1,A2,A3 +seacoast 01#A1 +search 01#A1,A0 +season 01#A1,A2,A0 +seasonality 01#A1,A2 +seat 01#A1,A2,A0 02#A1 03#A1,A2 +seclusion 01#A1,A0 +second 01#A1,A2,A3 02#A1,A2,A3 +secrecy 01#A1,A2 +secret 01#A1,A2,A0 +secretary 01#A1,A2,A3,A0 +section 01#A1,A2 +sector 01#A1,A2,A3 +security 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +seed 01#A1,A2,A4,A0 +seeker 01#A1,A2,A0 +seer 01#A1,A0 +segment 01#A1,A2,A3 02#A1,A3,A0 +segmentation 01#A1,A2,A0 +segregation 01#A1,A2,A0 +seige 01#A1,A2,A0 +seizure 01#A1,A2,A0 02#A1 +selection 01#A1,A2,A3,A0 02#A1,A2,A3 +sell 01#A1,A2,A3,A4,A0 +sell-off 01#A1,A2,A3,A4,A0 +seller 01#A1,A2,A3,A4,A0 02#A1,A0 +selling 01#A1,A2,A3,A4,A0 +semblance 01#A1 +seminar 01#A1,A2,A0 +senator 01#A1,A2,A4,A0 +sender 01#A1,A2,A0 +senior 01#A1,A0 +seniority 01#A1,A2 +sensation 01#A1,A2,A0 02#A1,A0 +sense 01#A1,A2,A0 02#A1,A2 03#A1,A2 04#A1,A0 +sensibility 01#A1,A2 +sensitivity 01#A1,A0 +sensor 01#A1,A2,A0 +sensuality 01#A1,A2 +sentence 01#A1,A2,A3,A0 02#A1,A0 +sentencing 01#A1,A2,A3,A0 +sentiment 01#A1,A2,A0 02#A1,A2,A0 +separation 01#A1,A2,A0 +sequel 01#A1,A2,A0 +sequence 01#A1,A3 +sequester 01#A1,A0 +sergeant 01#A1,A2,A3,A0 +series 01#A1,A3 02#A1,A2,A0 03#A1,A2 04#A1,A0 +seriousness 01#A1,A2 +servant 01#A1,A2,A0 +service 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +servicing 01#A1,A0 +session 01#A1,A2,AM,A0 +set 01#A1,A2,A3 02#A1,A2,A0 +setback 01#A1,A2,A3,A0 +setter 01#A1,A2,A0 +setting 01#A1,A2,A3 02#A1,A2,A0 +settlement 01#A1,A2,A3,A0 +settling 01#A1,A2,A3,A0 02#A1,A2,A3,A4,A0 +setup 01#A1,A2,A0 +severity 01#A1,A2,A3 +sex 01#A1,A2,A3 02#A1,A0 +shadow 01#A1,A0 02#A1,A0 03#A1,A0 +shah 01#A1,A2,A3,A0 +shake 01#A1,A0 02#A1,A2,A0 +shake-up 01#A1,A0 +shakeout 01#A1 +shambles 01#A1,A2,A0 +shame 01#A1,A2,A0 +shape 01#A1,A2,A3,A0 +shard 01#A1 +share 01#A1,A2,A3,A4,A0 +shareholder 01#A1,A2,A0 +shareholding 01#A1,A2,A0 +sharing 01#A1,A2,A0 +sharpness 01#A1,A2 +sheaf 01#A1,A3 +shedding 01#A1,A0 +sheet 01#A1,A3 02#A1,A3,A0 +sheik 01#A1,A2,A3,A0 +shelter 01#A1,A2,A3,A0 +sheriff 01#A1,A2,A3,A0 +shield 01#A1,A2,A0 +shift 01#A1,A2,A3,A0 02#A1,A2,A3 +shipment 01#A1,A2,A3,A4,A0 +shipper 01#A1,A2,A3,A0 +shipping 01#A1,A2,A3,A4,A0 +shiver 01#A1,A2,A0 +shock 01#A1,A2,A0 +shoo-in 01#A1,A2,A0 +shooting 01#A1,A2,A3,A0 02#A1,A0 +shopper 01#A1,A4,A0 +shopping 01#A1,A2,A4,A0 +shore 01#A1 +shoreline 01#A1 +shortage 01#A1,A2,A0 +shortcoming 01#A1,A2,A3 +shortcut 01#A1,A2,A3,A4 +shortfall 01#A1,A2,A3,A4,A0 +shot 01#A1,A0 02#A1,A2,A3,A0 03#A1,A2,A3,A0 04#A1,A0 05#A1 06#A1,A0 07#A1,A2,A0 08#A1,A2,A0 09#A1,A2,A0 +shoulder 01#A1 02#A1,A3,A0 +shouting 01#A1,A2,A0 +show 01#A1,A2,A3,A0 02#A1,A2,A0 +showcase 01#A1,A2,A0 +shower 01#A1,A2,A0 02#A0 +showing 01#A1,A2,A0 +shred 01#A1,A3 +shrine 01#A1,A2,A0 +shrinkage 01#A1,A2,A3,A4,A0 +shuffle 01#A1,A0 +shutdown 01#A1,A2,AM,A0 +shuttle 01#A1,A2,A3,A0 +sibling 01#A1,A3,A0 +side 01#A1 02#A1,A2,A3 03#A1,A2,A3 04#A1,A0 +sigh 01#A1,A2,A0 +sight 01#A1,A0 +sighting 01#A1,A0 +sign 01#A1,A2,A0 +signal 01#A1,A2,A3,A0 +signature 01#A1,A2,A0 02#A1,A0 +significance 01#A1,A2,A0 +signing 01#A1,A2,A0 +silence 01#A1,A2,A0 02#A1,A2 +similarity 01#A1,A2,A0 +simplicity 01#A1 +simulation 01#A1,A2,A0 +simulator 01#A1,A2,A0 +sin 01#A1,A0 +sincerity 01#A1,A0 +singer 01#A1,A2,A4,A0 +sinking 01#A1,A2,A3,A4,A5,A0 +sister 01#A1,A3,A0 +sitcom 01#A1,A0 +site 01#A1,A2,A3 02#A1 +situation 01#A1,A2,A3,A0 +size 01#A1,A2,A3 +skepticism 01#A1,A0 +sketch 01#A1,A2,A0 02#A1,A0 +skid 01#A1,A0 02#A1,A2,A3,A4 +skill 01#A1,A0 +skin 01#A1 +skirmish 01#A1,A2,A0 +skittishness 01#A1,A0 +skyline 01#A1,A2,A3 +slap 01#A1,A2,A0 +slapping 01#A1,A2,A0 +slash 01#A1,A2,A3,A4,A0 +slat 01#A1 +slate 01#A1,A2,A3 +slaughter 01#A1,A0 +slavery 01#A1,A2,A0 +slaying 01#A1,A0 +sledding 01#A1,A2,A3,A4,A0 +sleep 01#A1,A2,A0 +slew 01#A1,A3 +slice 01#A1,A2,A3,A0 +slide 01#A1,A0 02#A1,A2,A3,A4,AM +slinger 01#A1,A2,A0 +slip 01#A1,A2,A3,A4,AM 02#A1,A3 +slippage 01#A1,A2,A3,A4,AM +sliver 01#A1,A3 +slot 01#A1,A2,A0 +slouch 01#A1,A0 +slowdown 01#A1,A0 +slowing 01#A1,A3,A4,AM +sluggishness 01#A1,A2,A3 +slump 01#A1,A2,A3,A4,A0 +slur 01#A1,A0 +smattering 01#A1,A3 +smell 01#A1,A2,A3 +smelter 01#A1,A2,A0 +smidgin 01#A1,A3 +smile 01#A1,A2,A0 +smoke 01#A1,A0 +smoker 01#A1,A0 +smoking 01#A1,A0 +snafu 01#A1,A2,A3,A0 +snag 01#A1,A2,A0 +snapshot 01#A1,A0 +snippet 01#A1,A3 +snobbery 01#A0 +society 01#A1,A2,A3 +sociologist 01#A1,A2,A3,A0 +softener 01#A1,A0 +softening 01#A1,A0 +softness 01#A1,A2 +software 01#A1,A2,A3,A0 +soiree 01#A1,A2,A0 +solace 01#A1,A2,A0 +soldier 01#A1,A2,A3,A0 +solicitation 01#A1,A2,A0 +solidarity 01#A1,A0 +solution 01#A1,A2,A0 02#A1,A2,A3,A0 +son 01#A1,A3,A0 +song 01#A1,A0 +sophistication 01#A1,A2,A3 +sort 01#A1 02#A1 03#A1,A2,A0 +soul 01#A1 +soulmate 01#A1,A3,A0 +sound 01#A1,A2,A3 +soundness 01#A1,A2,A3 +soundtrack 01#A1,A0 +source 01#A1,A2,A3,A0 +souring 01#A1,A2,A0 +south 01#A1,A2 +southeast 01#A1,A2 +souvenir 01#A1,A2,A0 +sovereignty 01#A1 +space 01#A1,A2,A3 +span 01#A1,A2,A3 02#A1 +spanking 01#A1,A0 +spark 01#A1,A2,A3 +sparkle 01#A1,A2 +spasm 01#A1 02#A1 +spate 01#A1,A3 +speaker 01#A1,A2,A0 02#A1,A0 +spearhead 01#A1,A0 +special 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A0 +specialist 01#A1,A2,A3,A4,A0 +specialty 01#A1,A0 +species 01#A1 +specification 01#A1,A0 +specificity 01#A1,A2,A3 +specimen 01#A1,A0 +specter 01#A1,A2,A3,A0 +spectrum 01#A1,A3 +speculation 01#A1,A0 +speculator 01#A1,A0 +speech 01#A1,A2,A0 +speed 01#A1,A2 +speeding 01#A1,A0 +speedup 01#A1,A0 +spell 01#A1,A0 +spelling 01#A1,A2,A3 +spender 01#A1,A2,A3,A4,A0 +spending 01#A1,A2,A3,A4,A0 +sphere 01#A1,A2,A3,A0 +spice 01#A1,A2,A0 +spill 01#A1,A2,A0 +spin-off 01#A1,A2,A0 +spiral 01#A1,A2,A0 +spirit 01#A1,A2,A3 +split 01#A1,A2,A0 +spoiler 01#A1 +spokesman 01#A1,A2,A3,A0 +spokeswoman 01#A1,A2,A3,A0 +sponsor 01#A1,A2,A3,A0 +sponsorship 01#A1,A0 +spook 01#A1,A2,A3,A0 +spot 01#A1,A2,A3 02#A1,A2,A0 03#A1,A2,A3,A0 04#A1 +spouse 01#A1,A3,A0 +spread 01#A1,A2,A3 02#A1,A2 03#A1,A0 +spree 01#A1,A0 +spring 01#A1,A2,A3 +spurt 01#A1,A2 02#A1,A2,A3,A4 +squabble 01#A1,A2,A0 +squabbling 01#A1,A2,A0 +squad 01#A1,A2,A3 +squalor 01#A1,A2 +stab 01#A1,A0 +stability 01#A1,A2,A3 +stabilization 01#A1,A2,A0 +stable 01#A1,A3 +stack 01#A1,A3 +staff 01#A1,A2,A3,A0 +staffer 01#A1,A2,A0 +stage 01#A1,A3 02#A1,A2,A3 03#A1 +staging 01#A1,A0 +stagnation 01#A1 +stake 01#A1,A2,A0 02#A1,A2,A0 +stalwart 01#A1,A0 +stamp 01#A1,A0 +stampede 01#A1,A2,A0 +stance 01#A1,A2,A0 02#A1,A2 +stand 01#A1,A0 02#A1,A2,A0 03#A1,A3,A4,A0 04#A1,A2,A0 +standard 01#A1,A2,A3,A0 +standing 01#A1,A2,A3,A0 02#A1 +standoff 01#A1,A2,A0 +standpoint 01#A1,A2,A3 +standstill 01#A1,A2,A0 +staple 01#A1,A0 +star 01#A1,A3,A0 +stardom 01#A1,A2,A0 +start 01#A1,A2,A0 02#A1,A2,A3 +start-up 01#A1,A2,A0 +starter 01#A1,A2,A0 02#A1 +state 01#A1,A2,A3 02#A1 +statehood 01#A1,A2,A3 +statement 01#A1,A2,A3,A0 +station 01#A1,A2,A0 +statism 01#A1,A2,A3 +statistic 01#A1,A0 +statue 01#A1,A0 +stature 01#A1,A2,A0 +status 01#A1,A2 +statute 01#A1,A2,A3,A0 +stay 01#A1,A2,A3 02#A1,A0 +steadiness 01#A1,A2 +steering 01#A1,A2,A0 02#A1 +step 01#A1,A2,A0 02#A1 03#A1,A2 04#A1,A3 +stepmother 01#A1,A3,A0 +stereotype 01#A1,A2,A0 +sterility 01#A1,A2 +sterilization 01#A1,A0 +stewardship 01#A1,A0 +sticker 01#A1,A2,A0 +stickiness 01#A1,A2 +stickler 01#A1,A0 +stigma 01#A1,A0 +stimulation 01#A1,A2,A0 02#A1,A2,A3,A0 +stimulator 01#A1,A2,A3,A0 +stint 01#A1,A2,A0 +stipend 01#A1,A2,A3,A0 +stirring 01#A1,A2,A0 +stock 01#A1,A2,A3,A0 02#A1,A2,A3,A0 03#A1,A0 +stockbroker 01#A1,A2,A3,A4,A0 +stockholder 01#A1,A0 +stockholding 01#A1,A2,A3,A0 +stocking 01#A1,A2,A0 02#A1,A0 +stockpile 01#A1,A2,A3,A0 +stomach 01#A1,A0 +stop 01#A1,A2,A0 02#A1,A0 +stopgap 01#A1,A2,A3,A0 +stoppage 01#A1,A2,A0 +stopper 01#A1,A2,A0 02#A1,A2,A0 +storage 01#A1,A2,A0 +store 01#A1,A2,A0 02#A1,A2 03#A1,A2,A0 +storm 01#A1,A3 +story 01#A1,A3,A0 02#A1,A3 +storytelling 01#A1,A2,A0 +strain 01#A1 02#A1,A0 +strainer 01#A1,A2,A0 +strangeness 01#A1,A2 +stranger 01#A1,A0 +strategist 01#A1,A2,A3,A0 +strategy 01#A1,A0 +streak 01#A1,A3 +stream 01#A1,A2,A3 +streamlining 01#A1,A0 +street 01#A1 02#A1 +strength 01#A1,A2,A3 02#A1,A2,A0 +strengthening 01#A1,A2,A3,A4,A0 +stress 01#A1,A0 +stretch 01#A1,A2,A3,A4,A0 02#A1,A3,A0 +strife 01#A1,A2,A0 +strike 01#A1,A2,A0 02#A1,A2,A0 03#A1,AA,A0 +striker 01#A1,A2,A0 +string 01#A1,A3 02#A1,A2,A0 +strip 01#A1,A3 02#A1,A3,A0 03#A1 +stroke 01#A1,A2,A0 02#A1,A2,A0 +structure 01#A1,A2,A3 +struggle 01#A1,A2,A0 02#A1,A0 +student 01#A1,A0 +studiousness 01#A0 +study 01#A1,A2,A0 02#A1 03#A1,A2,A0 +stuff 01#A1,A2,A3 02#A1 +stumble 01#A1,A0 +stunt 01#A1,A2,A3,A4,A0 +style 01#A1,A2,A0 +stylist 01#A1,A2,A0 +subcommittee 01#A1,A2,A3 +subcontractor 01#A1,A2,A0 +subdirector 01#A1,A2,A3,A0 +subindustry 01#A1,A2,A3 +subject 01#A1,A2,A0 02#A1,A2,A3 03#A1 04#A1,A0 +sublicense 01#A1,A2,A0 +submission 01#A1,A2,A3,A0 +subpoena 01#A1,A2,A0 +subscriber 01#A1,A0 +subscription 01#A1,A0 +subset 01#A1 +subsidiary 01#A1 +subsidization 01#A1,A0 +subsidy 01#A1,A0 +substance 01#A1,A0 +substitute 01#A1,A3,A4,A0 +substitution 01#A1,A3,A4,A0 +subtitle 01#A1,A2,A3 +subtlety 01#A1,A2,A3 +subtraction 01#A1,A2,A0 +suburb 01#A1,A3,A0 +success 01#A1,A0 +succession 01#A1,A2,A0 02#A1,A2 +successor 01#A1,A2,A0 +suffering 01#A1,A0 +suggestion 01#A1,A2,A0 +suicide 01#A1,A2,A0 +suit 01#A1,A2,A3,A0 02#A1,A0 03#A1,A0 +suitor 01#A1,A2 +sultan 01#A1,A2,A3,A0 +sum 01#A1,A3 +summary 01#A1,A2,A0 +summer 01#A1,A2,A3 02#A1 +summit 01#A1,A2,AM,A0 +superagent 01#A1,A2,A3,A4,A0 +superior 01#A1,A0 +superiority 01#A1,A2,A3 +superstar 01#A1,A0 +supervision 01#A1,A0 +supervisor 01#A1,A0 02#A1,A2,A3,A0 +supplement 01#A1,A2,A0 +supplier 01#A1,A2,A0 +supply 01#A1,A2,A3,A0 +support 01#A1,A0 +supporter 01#A1,A0 +suppository 01#A1,A2,A0 +suppressant 01#A1,A0 +suppression 01#A1,A0 +suppressor 01#A1,A0 +supremacy 01#A1,A2 +surf 01#A1 +surface 01#A1 +surge 01#A1,A2,A3,A4,AM +surgery 01#A1,A2,A3,A0 +surplus 01#A1,A2,A3,A4,A0 +surprise 01#A1,A2,A0 +surrender 01#A1,A2,A0 +surrogate 01#A1,A3,A4,A0 +surtax 01#A1,A2,A3,A0 +surveillance 01#A1,A0 +survey 01#A1,A2,A3,A0 +survival 01#A1,A0 +survivor 01#A1,A0 02#A1,A2,A0 +suspect 01#A1,A2,A0 +suspense 01#A1,A2,A0 +suspension 01#A1,A2,A3,A4,A0 02#A1,A2,A0 +suspicion 01#A1,A2,A0 +sustainability 01#A1,A0 +sustenance 01#A1,A0 +swagger 01#A1,A2,A3 +swap 01#A1,A2,A3,A4,A0 +swarm 01#A1,A3 +swat 01#A1,A2,A0 +swath 01#A1,A3 +sway 01#A1,A0 +swaying 01#A1,A2,A0 +swearing 01#A1,A2,A0 +swearing-in 01#A1,A2,A0 +sweat 01#A1,A0 +sweep 01#A1,A2,A3,A4,A5,A0 02#A1,A2,A3 03#A1,A2,A3,A0 04#A1,A3 +sweeper 01#A1,A2,A3,A0 +sweetheart 01#A1,A3,A0 02#A1,A2 +swell 01#A1,A2,A3,A4,A0 +swig 01#A1,A2,A0 +swing 01#A1,A0 02#A1,A2,A3,A4 +swirl 01#A1,A3 +switch 01#A1,A2,A3,A0 +switcher 01#A1,A2,A3,A0 +swoon 01#A1,A2,A0 +symbol 01#A1,A2 +symmetry 01#A1,A2 +sympathizer 01#A1,A0 +sympathy 01#A1,A0 +symptom 01#A1,A2,A0 +sync 01#A1,A2,A0 +syndicate 01#A1,A2,A3 +syndication 01#A1,A2,A0 +syndicator 01#A1,A2,A0 +syndrome 01#A1,A2,A3 +synergy 01#A1,A2,A0 +synthesis 01#A1,A2,A3,A4,A0 +synthesizer 01#A1,A2,A3,A4,A0 +system 01#A1,A0 +tab 01#A1,A0 02#A1,A2,A3,A4,A0 +table 01#A1,A0 02#A1,A2,A0 03#A1 04#A0 +taboo 01#A1,A2,A0 +tactic 01#A1,A0 +tag 01#A1,A2,A3 +tail 01#A1 +tailspin 01#A1,A2 +taint 01#A1,A2,A0 +take 01#A1,A2,A3,A0 +takeover 01#A1,A0 +taker 01#A1,A2,A0 +taking 01#A1,A0 02#A1,A2,A0 +tale 01#A1,A0 +talent 01#A1,A0 02#A1,A2,A0 +talk 01#A1,A2,A0 +tally 01#A1,A0 +tangle 01#A1,A3 +tape 01#A1,A0 +taper 01#A1,A0 +taping 01#A1,A0 +target 01#A1,A2,A3,A0 02#A1,A0 +tariff 01#A1,A2,A3,A0 +task 01#A1,A0 +taste 01#A1,A2,A3,A0 02#A1,A2,A3 +tax 01#A1,A2,A3,A0 +taxation 01#A1,A2,A3,A0 +teacher 01#A1,A2,A3,A0 +teaching 01#A1,A2,A0 +team 01#A1,A2,A3 +teammate 01#A1,A2,A3,A0 +teaspoon 01#A1,A3 +tech 01#A1,A2,A3 +technician 01#A1,A2,A3,A0 +technique 01#A1,A0 +technocrat 01#A1,A2,A3,A0 +technologist 01#A1,A2,A3,A0 +technology 01#A1,A0 +telecast 01#A1,A2,A3,A0 +telegram 01#A1,A2,A0 +televising 01#A1,A0 +teller 01#A1,A2,A3,A0 +temblor +temperament 01#A1,A2 +temperature 01#A1,A2,A3 +temple 01#A1 +tempo 01#A1,A2,A3 +temptation 01#A1,A2,A3,A0 +tenant 01#A1,A2,A3,A4,A0 +tendency 01#A1,A2 +tender 01#A1,A2,A3,A0 +tenet 01#A1,A2,A0 +tenor 01#A1,A2,A3 +tens 01#A1,A3 +tension 01#A1,A2,A0 02#A1,A2 +tenth 01#A1,A3 +tenure 01#A1,A2,A0 +term 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2,A0 05#A1,A2 06#A1,A0 07#A1,A2 +termination 01#A1,AM,A0 02#A1,A2,A0 +terrain 01#A1,A2,A3 +territory 01#A1,A0 +terrorism 01#A1,A2,A0 +test 01#A1,A2,A0 +testimonial 01#A1,A2,A0 +testimony 01#A1,A2,A0 +testing 01#A1,A2,A0 +texasness 01#A1,A2,A3 +text 01#A1,A0 +texture 01#A1,A2,A3 +thank 01#A1,A2,A0 +thaw 01#A1,A0 +theft 01#A1,A2,A0 +theme 01#A1,A0 02#A1,A2,A3 +theorist 01#A1,A2,A0 +theory 01#A1,A0 +therapist 01#A1,A2,A3,A0 +therapy 01#A1,A2,A3,A0 +thicket 01#A1,A3 +thickness 01#A1,A2,A3 +thief 01#A1,A2,A0 +thing 01#A1,A2,A3 +thinker 01#A1,A2,A0 +thinking 01#A1,A2,A0 +third 01#A1,A3 +thirst 01#A1,A0 +thought 01#A1,A2,A0 +thousands 01#A1,A3 +thrall 01#A1,A2,A0 +thread 01#A1,A0 +threat 01#A1,A2,A0 +threshold 01#A1,A2,A3 +thrill 01#A1,A0 +thriller 01#A1,A0 +throat 01#A1 +thrower 01#A1,A2,A0 +thrust 01#A1,A2,A3 02#A1,A2,A0 +thumb 01#A1 +ticket 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 +tidbit 01#A1,A3 +tide 01#A1,A2 +tie 01#A1,A2,A3,A0 02#A1 +tie-up 01#A1,A2 +tightener 01#A1,A0 +tightening 01#A1,A0 +tilt 01#A1,A2,A0 +tilth 01#A1,A0 +time 01#A1,A2,A3,A0 02#A1 03#A1,A2 04#A1,A2 05#A1,A2 06#A1 07#A1,A2 08#A1,A0 +timetable 01#A1,A2,A0 +timidity 01#A1,A0 +timing 01#A1,A2,A0 +tin 01#A1,A3 +tinge 01#A1 +tinkering 01#A1,A0 +tip 01#A1,A2,A0 02#A1 03#A1,A2,A0 +tip-off 01#A1,A2,A0 +tipper 01#A1,A2,A0 +tipster 01#A1,A2,A0 +tissue 01#A1 +titan 01#A1,A0 +title 01#A1,A2,A0 02#A1,A2,A3,A0 03#A1,A2,A3 04#A1 +toast 01#A1,A0 +toddler 01#A1,A3,A0 +toe 01#A1 +toehold 01#A1,A2,A0 +toiling 01#A1,A0 +tolerance 01#A1,A0 +toll 01#A1,A2,A3,A0 02#A1,A0 +tomb 01#A1,A3,A0 +ton 01#A1,A3 +tone 01#A1,A2,A3 +tongue 01#A1 +tonic 01#A1,A2,A3,A0 +tonnage 01#A1,A2,A3 +tool 01#A1,A0 +tooling 01#A1,A2,A0 +tooth 01#A1 02#A1,A2 +top 01#A1 02#A1 03#A1 +topic 01#A1,A2,A0 02#A1 +tormentor 01#A1,A2,A0 +torrent 01#A1,A3 +tosser 01#A1,A0 +total 01#A1,A2,A3 +touch 01#A1,A3 02#A1,A2,A0 03#A1,A2,A0 04#A1,A0 05#A1,A0 06#A1,A2 +toughness 01#A1,A2 +tour 01#A1,A0 +tourism 01#A1,A0 +tourist 01#A1,A0 +tournament 01#A1,A2,A0 +town 01#A1 02#A1 +toxicity 01#A1,A2 +trace 01#A1 +track 01#A1,A0 02#A1,A0 03#A1,A0 +tracking 01#A1,A2,A3,A0 +tract 01#A1,A3 +trade 01#A1,A2,A3,A4,A0 02#A1,A0 +trade-in 01#A1,A2,A0 +trademark 01#A1,A0 +trader 01#A1,A2,A3,A0 +trading 01#A1,A2,A3,A4,A0 +tradition 01#A1,A0 +traditionalist 01#A1,A0 +traffic 01#A1,A2,A0 02#A1,A0 +trafficker 01#A1,A2,A3,A4,A0 +trafficking 01#A1,A0 02#A1,A2,A0 +trail 01#A1,A3 +trainer 01#A1,A2,A0 +training 01#A1,A2,A0 +trait 01#A1,A2,A3 +transaction 01#A1,A2,A3,A0 +transcript 01#A1,A0 +transfer 01#A1,A2,A3,A0 +transformation 01#A1,A2,A3,A0 +transfusion 01#A1,A2,A3,A0 +transition 01#A1,A2,A3,A0 +translation 01#A1,A2,A3,A0 +translator 01#A1,A2,A3,A0 +transmission 01#A1,A2,A0 +transmitter 01#A1,A2,A0 +transplant 01#A1,A2,A3,A0 +transplantation 01#A1,A2,A3,A0 +transport 01#A1,A2,A3,A0 +transportation 01#A1,A2,A3,A0 +transporter 01#A1,A2,A3,A0 +trap 01#A1,A2,A3,A0 +trapping 01#A1,A2,A3 +trauma 01#A1,A2,A0 +travail 01#A1,A0 +travel 01#A1,A0 +traveler 01#A1,A0 +treasurer 01#A1,A2,A3,A0 +treasury 01#A1,A2,A0 +treatment 01#A1,A2,A0 02#A1,A2,A3,A0 03#A1,A2,A0 04#A1,A2,A0 +treaty 01#A1,A2,A0 +tree 01#A1,A2,A3 +tremblor +tremor 01#A1,A0 +trend 01#A1,A2,A3 +trepidation 01#A1,A0 +trespass 01#A1,A0 +triage 01#A1,A0 +trial 01#A1,A0 02#A1,A2,A3,A0 +tribunal 01#A1,A2,A3,A0 +tribute 01#A1,A2 +trick 01#A1,A0 02#A1,A0 +trickery 01#A1,A0 +trigger 01#A1,A2,A3,A0 +trillion 01#A1,A3 +trip 01#A1,A2,A0 +triumph 01#A1,A2,A0 +troop 01#A1,A2,A3 +trophy 01#A1,A2,A3,A0 +trouble 01#A1,A2,A0 +trough 01#A1,A2,A3 +truce 01#A1,A2,A0 +truculence 01#A2,A0 +trudge 01#A1,A0 +trunk 01#A1 +trust 01#A1,A2,A3 02#A1,A0 +trustee 01#A1,A2,A3,A0 +truth 01#A1,A2 +try 01#A1,A0 +tube 01#A1,A3 02#A1 +tuition 01#A1,A2,A3,A0 +tumble 01#A1,A2,A3,A4,AM,A0 +tumor 01#A1,A2,A0 +tumult 01#A1,A2,A0 +tune 01#A1,A0 02#A1 03#A1,A2 +tuning 01#A1,A2,A0 +turbulence 01#A1,A2,A3 +turf 01#A1,A0 +turmoil 01#A1,A0 +turn 01#A1,AM,A0 02#A1,A0 +turn-on 01#A1,A2,A0 +turnabout 01#A1,A0 +turnaround 01#A1,A0 +turnover 01#A1,A2,A0 02#A1,A2 +twist 01#A1,A0 +two-hundredth 01#A1,A3 +tycoon 01#A1,A0 +type 01#A1,A2 +ubiquity 01#A1,A2 +ulcer 01#A1,A2,A0 +ultimatum 01#A1,A2,A0 +umbrella 01#A1,A2,A3,A0 +unanimity 01#A1 +unavailability 01#A1 +unawareness 01#A1,A0 +unbanning 01#A1,A2,A0 +uncertainty 01#A1,A2,A0 +uncle 01#A1,A3,A0 +undercurrent 01#A1,A2,A3 +undergirding 01#A1,A0 +underside 01#A1 +understanding 01#A1,A2,A0 02#A1,A2,A0 +understatement 01#A1,A2,A0 +undertaking 01#A1,A2,A0 +undertone 01#A1,A2,A3 +underwriter 01#A1,A0 +underwriting 01#A1,A0 +undoing 01#A1,A2,A0 +unease 01#A1,A0 +uneasiness 01#A1,A0 +unemployment 01#A1,A2,A3 +unhappiness 01#A1,A0 +unification 01#A1,A2,A3,A0 +unifier 01#A1,A2,A3,A0 +uniformity 01#A1,A2,A3 +union 01#A1,A2,A3 02#A1,A2,A3,A0 +unit 01#A1 02#A1,A3 03#A1,A2,A0 04#A1,A2,A3 05#A1,A2,A3 +unitholder 01#A1,A2,A0 +unity 01#A1,A2,A3,A0 +universe 01#A1 +unknown 01#A1,A2,A0 +unpopularity 01#A1,A2 +unraveling 01#A1,A0 +unrest 01#A1,A2,A0 +unsettlement 01#A1,A2,A0 +unsteadiness 01#A1,A2 +unveiling 01#A1,A2,A0 +unwillingness 01#A1,A0 +up 01#A1,A2,A3,A4,A0 +update 01#A1,A2,A0 +upgrade 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +upgrading 01#A1,A2,A3,A4,A0 +upheaval 01#A1 +upholstery 01#A1 +uprising 01#A1,A2,A3,A4,AM +uproar 01#A1,A0 +ups-and-down 01#A1,A2,A3,A0 +upset 01#A1,A2,A0 +upshot 01#A1,A0 +upsurge 01#A1,A2,A3,A4,AM +uptick 01#A1,A2,A3,A4,A0 +upturn 01#A1,A2,A3,A4,A0 +urge 01#A1,A2,A0 +urgency 01#A1,A2,A3 +urging 01#A1,A2,A0 +usability 01#A1,A2,A0 +usage 01#A1,A2,A0 +use 01#A1,A2,A0 +usefulness 01#A1,A2 +user 01#A1,A2,A0 +usher 01#A1,A2,A3,A0 +usurpation 01#A1,A0 +utility 01#A1,A2,A0 02#A1,A2,A0 +utilization 01#A1,A0 +utopia 01#A1,A2,A3 +utterance 01#A1,A0 +vacancy 01#A1,A2 +vacation 01#A1,A0 +vacationer 01#A1,A0 +vaccine 01#A1,A2,A3,A0 +vagary 01#A1,A2 +validity 01#A1,A2 +valuation 01#A1,A2,A0 +value 01#A1,A2,A3 +valve 01#A1,A2,A0 +vanity 01#A1,A2,A3 +variable 01#A1,A2,A3,A4,A5,A0 +variation 01#A1,A2,A3,A4,A5,A0 +variety 01#A1,A2,A3,A4,A5,A0 02#A1 +vassal 01#A1,A2,A3,A0 +vault 01#A1,A2,A0 +vehemence 01#A1,A0 +vehicle 01#A1,A0 +vein 01#A1 +velocity 01#A1,A2,A3 +vendetta 01#A1,A2,A0 +vendor 01#A1,A0 +vengeance 01#A1,A2,A0 02#A1 +ventilation 01#A1,A0 +venture 01#A1,A2,A0 02#A1,A0 +veracity 01#A1,A2 +verdict 01#A1,AA,A2,A0 02#A1,A2,A0 +verge 01#A1,A2 +verification 01#A1,A0 +version 01#A1,A0 +vessel 01#A1,A2 +veteran 01#A1,A0 02#A1,A2,A0 +veto 01#A1,A2,A0 +viability 01#A1,A2,A3 +vial 01#A1,A3 +vicar 01#A1,A2,A3,A0 +vicinity 01#A1,A2,A3 +vicissitude 01#A1,A2,A3,A4,A5,A0 +victim 01#A1,A2,A0 +victor 01#A1,A2,A3,A4,A0 +victory 01#A1,A2,A3,A4,A0 +video 01#A1,A3,A0 +view 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A0 +viewer 01#A1,A0 +viewership 01#A1,A2,A3 +viewpoint 01#A1,A2,A0 +vigil 01#A1,A0 +vignette 01#A1,A0 +vigor 01#A1,A2,A3 +village 01#A1,A3 +vindication 01#A1,A2,A0 +vintage 01#A1,A2,A3 +violation 01#A1,A0 +violence 01#A1,A0 02#A1,A2,A3 +virility 01#A0 +virtue 01#A1,A2,A3 +virtuosity 01#A1,A2 +virulence 01#A1,A2 +virus 01#A1,A2,A3 +visa 01#A1,A2,A0 +visibility 01#A1,A2 +vision 01#A1,A0 +visionary 01#A1,A0 +visit 01#A1,A0 +visitor 01#A1,A0 +vitality 01#A1,A2 +vogue 01#A1,A2 +voice 01#A1,A2,A3 02#A1,A0 +volatility 01#A1,A2,A3 +volley 01#A1,A3 +volume 01#A1,A2,A3 02#A1,A2,A3 03#A1,A2,A3 +volunteer 01#A1,A2,A0 +volunteerism 01#A1,A2,A0 +vote 01#A1,AA,A2,A0 02#A1,A2,A3 +voter 01#A1,AA,A2,A0 +voting 01#A1,AA,A2,A0 +voucher 01#A1,A2,A0 +vow 01#A1,A0 +voyage 01#A1,A2,A3 +vulnerability 01#A1,A2 +wad 01#A1,A3 +waffle 01#A1,A0 +wage 01#A1,A2,A3,A0 +waiver 01#A1,A2,A0 +wake 01#A1,A2,A0 +walk 01#A1,AA,A0 02#A1,A0 +walkout 01#A1,A0 +wall 01#A1 +war 01#A1,A0 +ward 01#A1,A2 +warden 01#A1,A2,A3,A0 +ware 01#A1,A2,A3,A4,A0 +warfare 01#A1,A0 +warm-up 01#A1,A2,A0 +warming 01#A1,A0 +warning 01#A1,A2,A0 +warrant 01#A1,A0 +warranty 01#A1,A2,A0 +warren 01#A1,A3 +wash 01#A1,A2,A3,A4,A0 +waste 01#A1,A2,A0 +wasting 01#A1,A2,A0 +watch 01#A1,A0 +watchdog 01#A1,A2,A3,A4,A0 +watcher 01#A1,A0 +watchman 01#A1,A2,A3,A0 +watercolor 01#A1,A0 +watt 01#A1,A3 +wave 01#A1,A3 02#A1,A0 03#A1,A2,A3 +wavelength 01#A1,A2,A3 +way 11#A1 01#A1,A2,A0 12#A1 02#A1,A2,A3,A0 13#A1 03#A1 04#A1,A0 05#A1,A2,A3 06#A1,A2,A0 07#A1,A0 08#A1,A2,A0 09#A1,A2,A3,A0 10#A1,A2 +weakening 01#A1,A2,A3,A4,A0 +weakness 01#A1,A2,A3 +wealth 01#A1,A2,A3 02#A1,A3 +weapon 01#A1,A2,A0 +weariness 01#A1,A0 +weather 01#A1,A2,A3 +web 01#A1,A3 +wedding 01#A1,A2,A0 +wedge 01#A1,A2,A0 02#A1 +week 01#A1,A2,A3 02#A1 03#A1,A2,A3 +weekend 01#A1,A2,A3 +weight 01#A1,A2,A3 02#A1,A2,A0 +weighting 01#A1,A2,A0 +welfare 01#A1,A2,A3 +well 01#A1 02#A1,A2,A3,A0 +welter 01#A1,A3 +west 01#A1,A2 +whereabouts 01#A1 +whim 01#A1,A0 +whimper 01#A1,A2,A0 +whipping 01#A1,A2,A0 +whirl 01#A1,A3 +whirlwind 01#A1,A3 +whisper 01#A1,A2,A0 +whitewash 01#A1,A0 +whiz 01#A1,A2,A0 +whole 01#A1,A3 +wholesaler 01#A1,A2,A3,A4,A0 +whoring 01#A1,A0 +widening 01#A1,A2,A3,A4,A0 +widow 01#A1,A3,A0 +wife 01#A1,A3,A0 +will 01#A1,A0 02#A1,A2,A3 +willingness 01#A1,A0 +win 01#A1,A2,A3,A4,A5,A0 +wind 01#A1,A2,A3,A0 02#A1,A2 +windfall 01#A1,A2,A3,A0 +window 01#A1 02#A1,A0 03#A1,A0 +wing 01#A1 02#A1,A2,A3 +wingbeat 01#A1,A0 +winner 01#A1,A2,A3,A4,A5,A0 +wiretap 01#A1,A0 +wisdom 01#A1,A0 +wish 01#A1,A2,A0 +wit 01#A1,A2,A3 +withdrawal 01#A1,A2,A0 +witness 01#A1,A0 02#A1,A2,A3,A0 +wizard 01#A1,A2,A0 +woe 01#A1,A2,A3 +wonder 01#A1,A0 +word 01#A1,A2,A0 +work 01#A1,A2,A3,A4,A5,A0 02#A1 03#A1,A3,A0 04#A1,A2 +workbook 01#A1,A0 +worker 01#A1,A2,A3,A4,A0 +working 01#A1,A2,A3 +workout 01#A0 +worksheet 01#A1,A0 +workweek 01#A1,A2,A3 +world 01#A1,A2,A3 +worm 01#A1,A2,A3 +worry 01#A1,A0 +worsening 01#A1,A2,A0 +worship 01#A1,A0 +worth 01#A1,A2,A3 02#A1,A3 +worthiness 01#A1,A0 +wound 01#A1,A2,A0 +wrangling 01#A1,A2,A0 +wrath 01#A1,A0 +wreck 01#A1,A2,A0 +wreckage 01#A1,A2,A0 +wrinkle 01#A1,A2,A3,A0 +wrist 01#A1 +write-down 01#A1,A2,A3,A4,A0 +write-off 01#A1,A2,A0 +writer 01#A1,A2,A0 +writing 01#A1,A2,A0 +wrongdoing 01#A1,A2,A3,A4,A0 +yarn 01#A1,A0 +year 01#A1,A2,A3,A0 02#A1 03#A1 04#A1,A2 05#A1 +yearning 01#A1,A0 +yen 01#A1,A3 +yield 01#A1,A2,A0 +youth 01#A1 02#A1,A2,A3 +zenith 01#A1,A2,A3 +zip 01#A1,A2,A3 +zone 01#A1,A2,A3 diff --git a/srl/src/main/resources/Verb.cross-predicate.modifiers b/srl/src/main/resources/Verb.cross-predicate.modifiers new file mode 100644 index 000000000..37e5a1521 --- /dev/null +++ b/srl/src/main/resources/Verb.cross-predicate.modifiers @@ -0,0 +1,10 @@ +AM-ADV put,A2 +AM-CAU ask,R-A1 impeach,A2 +AM-DIR inflate,A4 romp,A1 slip,A2 wiggle,A1 +AM-DIS hope,A1 +AM-EXT tumble,A2 +AM-LOC buckle,A2 call,A1 gather,R-AM-LOC grimace,A1 incorporate,A2 mire,A2 send,A1 work,A1 +AM-MNR advertise,A1 ask,R-A1 detect,A2 do,A3 drain,A1 fall,A2 feast,A1 hope,A1 look,A1 package,A3 sell,A3 start,A2 stay,A3 structure,A2 vote,A2 +AM-PNC buy,A4 design,A2 finance,A1 pay,A3 review,A2 sell,A4 use,A2 wonder,R-A1 +AM-REC market,A1 +AM-TMP arrest,A1 flee,R-AM-TMP imprison,A2 lower,A2 spend,A1 diff --git a/srl/src/main/resources/Verb.legal.arguments b/srl/src/main/resources/Verb.legal.arguments new file mode 100644 index 000000000..2aedb962a --- /dev/null +++ b/srl/src/main/resources/Verb.legal.arguments @@ -0,0 +1,3318 @@ +abandon 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +abate 01#A1,A0 +abdicate 01#A1,A0 +abduct 01#A1,A0 +abet 01#A1,A0 +abide 01#A1,A0 +abolish 01#A1,A0 +abominate 01#A1,A0 +abort 01#A1,A0 +abound 01#A1,A2 +abridge 01#A1,A0 +absent 01#A1,A2,A0 +absolve 01#A1,A2,A0 +absorb 01#A1,A0 +abstain 01#A1,A0 +abuse 01#A1,A0 +accede 01#A1,A0 +accelerate 01#A1,A2,A3,A4,A0 +accent 01#A1,A2,A0 +accept 01#A1,A2,A3,A0 +access 01#A1,A0 ble.02#A1,A0 +acclaim 01#A1,A2,A0 +accommodate 01#A1,A0 +accompany 01#A1,A3,A4,A0 +accomplish 01#A1,A0 +accord 01#A1,A2,A0 +account 01#A1,A2,A0 +accrete 01#A1,A0 +accrue 01#A1,A0 +accumulate 01#A1,A3,A0 +accuse 01#A1,A2,A0 +accustom 01#A1,A2,A0 +ace 01#A1,A0 +ache 01#A0 +achieve 01#A1,A0 +acidify 01#A1,A0 +acknowledge 01#A1,A2,A3,A0 +acquaint 01#A1,A2,A0 +acquiesce 01#A1,A0 +acquire 01#A1,A2,A3,A4,A0 +acquit 01#A1,A2,A0 +act 01#A1,A0 02#A1,A0 +activate 01#A1,A0 +adapt 01#A1,A2,A3,A0 +add 01#A1,A0 02#A1,A2,A0 03#A1,A2,A4,AM 04#A1,A2 +addict 01#A1,A2,A0 +address 01#A1,A2,A0 02#A1,A2,A0 +adhere 01#A1,A2,A0 02#A1,A0 +adjourn 01#A1,A0 +adjudicate 01#A1,A0 +adjust 01#A1,A2,A3,A4,A0 +administer 01#A1,A2,A0 +admire 01#A1,A0 +admit 01#A1,A2,A0 02#A1,A2,A0 +admonish 01#A1,A2,A0 02#A1,A2,A0 +adopt 01#A1,A2,A0 +adorn 01#A1,A2,A0 +advance 01#A1,A2,A3,A4,A0 +advertise 01#A1,A0 +advise 01#A1,A2,A0 +advocate 01#A1,A2,A0 +affect 01#A1,A2,A0 02#A1,A0 +affiliate 01#A1,A2,A0 +affirm 01#A1,A2,A0 +afflict 01#A1,A2,A0 +afford 01#A1,A0 02#A1,A2,A0 +age 01#A1,A2,A0 +aggravate 01#A1,A2,A0 +agitate 01#A1,A0 +agonize 01#A1,A2 +agree 01#A1,A2,A0 +aid 01#A1,A2,A0 +ail 01#A1,A0 +aim 01#A1,A0 02#A1,A2,A0 +air 01#A1,A2,A0 +airlift 01#A1,A2,A3,A0 +alarm 01#A1,A0 +alert 01#A1,A2,A0 +alienate 01#A1,A2,A0 +align 01#A1,A2,A0 +allay 01#A1,A0 +allege 01#A1,A2,A0 +alleviate 01#A1,A0 +allocate 01#A1,A2,A0 +allow 01#A1,A0 02#A1,A2,A0 03#A1,A0 +allude 01#A1,A0 +ally 01#A1,A2,A0 +alter 01#A1,A2,A3,A0 +alternate 01#A1,A2,A0 +amalgamate 01#A1,A0 +amass 01#A1,A3,A0 +amaze 01#A1,A2,A0 +amble 01#A0 +ambush 01#A1,A0 +amend 01#A1,A2,A0 +americanize 01#A1,A2,A0 +amortize 01#A1,A2,A0 +amount 01#A1,A2 02#A1,A2 +amplify 01#A1,A2,A0 +amuse 01#A1,A2,A0 +analyze 01#A1,A0 +anchor 01#A1,A0 +anger 01#A1,A0 +animate 01#A1,A0 02#A1,A0 +announce 01#A1,A2,A3,A0 +annoy 01#A1,A2,A0 +anoint 01#A1,A0 +answer 01#A1,A2,A0 +antagonize 01#A1,A0 +anticipate 01#A1,A0 02#A1,A0 +apologize 01#A1,A2,A0 +appall 01#A1,A0 +appeal 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 +appear 01#A1 02#A1,A2 +appease 01#A1,A0 +append 01#A1,A2,A0 +applaud 01#A1,A2,A0 +apply 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A3,A0 04#A1,A0 +appoint 01#A1,A2,A0 +appraise 01#A1,A2,A0 02#A1,A0 +appreciate 01#A1,A2,A3,A4 02#A1,A0 +apprise 01#A1,A2,A0 +approach 01#A1,A2 02#A1,A2,A0 +appropriate 01#A1,A0 +approve 01#A1,A2,A0 +approximate 01#A1,A0 +arbitrage 01#A1,A0 +arbitrate 01#A1,A0 +arch 01#A1,A0 +argue 01#A1,A2,A0 02#A1,A2,A0 +arise 01#A1,A2 +arm 01#A1,A2,A0 +arouse 01#A1,A0 +arrange 01#A1,A2,A3,A4,A0 +arrest 01#A1,A2,A0 02#A1,A0 +arrive 01#A1,A2,A3,A4 +articulate 01#A1,A2,A0 +ascend 01#A1,A0 +ascribe 01#A1,A2,A0 +ask 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A2,A3,A0 +aspire 01#A1,A0 +assail 01#A1,A2,A0 +assassinate 01#A1,A0 +assemble 01#A1,A2,A4,A0 02#A1,A2,A0 +assert 01#A1,A0 02#A1,A2,A3,A0 03#A1,A2,A3,A0 +assess 01#A1,A2,A0 +assign 01#A1,A2,A0 +assimilate 01#A1,A2,A0 +assist 01#A1,A2,A0 +associate 01#A1,A2,A0 +assuage 01#A1,A0 +assume 01#A1,A2,A3,A4,A0 02#A1,A2,A0 +assure 01#A1,A2,A0 +astound 01#A1,A0 +atone 01#A1,A0 +attach 01#A1,A2,A0 +attack 01#A1,A2,A0 +attain 01#A1,A0 +attempt 01#A1,A0 +attend 01#A1,A0 +attest 01#A1,A0 +attract 01#A1,A2,A3,A0 +attribute 01#A1,A2,A0 +auction 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A4,A0 +audit 01#A1,A0 +audition 01#A1,A2,A0 +augment 01#A1,A2,A0 +author 01#A1,A0 +authorize 01#A1,A2,A3,A0 +autograph 01#A1,A0 +automate 01#A1,A0 +avenge 01#A1,A0 +aver 01#A1,A0 +average 01#A1,A2,A3,A0 +avert 01#A1,A2,A3,A0 +avoid 01#A1,A0 +await 01#A1,A2,Am +awake 01#A1,A2 +awaken 01#A1,A2,A0 +award 01#A1,A2,A0 +back 01#A1,A0 02#A1,A0 03#A1,A0 04#A1,A0 06#A1,A0 08#A1,A0 09#A1,A2,A0 10#A1 +backfire 01#A1,A2 +backpedal 01#A1,A0 +backslap 01#A0 +bail 01#A1,A0 02#A1,A2,A0 +balance 01#A1,A2,A0 +balk 01#A1,A0 +balloon 01#A0 02#A1,A2,A0 +ballyhoo 01#A1,A0 +ban 01#A1,A2,A0 +band 01#A1,A2,A3,A0 +bandy 01#A1,A0 +bang 01#A0 02#A1,A2,A0 +banish 01#A1,A2,A0 +bank 01#A1,A2,A0 02#A1,A0 +bankroll 01#A1,A2,A0 +bankrupt 01#A1,A0 +bar 01#A1,A2,A3,A0 +barb 01#A1,A0 +bare 01#A1,A0 +bargain 01#A1,A2,A0 +barge 01#A1,A0 +bark 01#A1,A2,A0 02#A0 +barrel 01#A1,A2 +barter 01#A1,A2,A3,A0 +base 01#A1,AM 02#A1,A2,A0 +bash 01#A1,A0 +bat 01#A1,A0 02#A1,A0 03#A1,A0 +bathe 01#A1,A0 +batter 01#A1,A2,A0 +battle 01#A1,A2,A0 +be 01#A1,A2 02#A1 +beam 01#A1,A0 02#A1,A0 +bear 01#A1,A2,A3,A0 02#A1,A0 03#A1 04#A1 05#A1,A0 06#A1,A2 +beat 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2,A0 05#A1,A2,A0 +become 01#A1,A2 +bedevil 01#A1,A2,A0 +beef 01#A1,A2,A3,A0 +beep 01#A1,A0 02#A1,A0 +befall 01#A1,A0 +befriend 01#A1,A0 +befuddle 01#A1,A0 +beg 01#A1,A2,A0 +beget 01#A1,A0 +begin 01#A1,A2,A0 +behave 01#A1,A0 +behead 01#A1,A0 +beleaguer 01#A1,A0 +belie 01#A1,A0 +believe 01#A1,A0 +belittle 01#A1,A0 +bellow 01#A1,A2,A0 +belly-flop 01#A1 +belong 01#A1,A0 +bemoan 01#A1,A0 +bend 01#A1,A0 02#A1,A0 +benefit 01#A1,A0 +bequeath 01#A1,A2,A0 +berate 01#A1,A2,A0 +beset 01#A1,A0 +besiege 01#A1,A2,A0 +best 01#A1,A2,A0 +bestow 01#A1,A2,A0 +bet 01#A1,A2,A3,A0 +betray 01#A1,A0 +better 01#A1,A0 +beware 01#A1,A0 +bewilder 01#A1,A2,A0 +bewitch 01#A1,A2,A0 +bias 01#A1 +bicker 01#A1,A0 +bid 01#A1,A2,A0 02#A1,A2,A3,A0 +bifurcate 01#A1,A0 +bike 01#A1,A0 +bill 01#A1,A2,A3,A0 02#A1,A2,A0 +billow 01#A1,A2 +bind 01#A1,A2,A3,A0 02#A1 03#A1,A2 +bite 01#A1,A0 +blab 01#A1,A2,A0 +black 01#A0 02#A1,A0 +blacken 01#A1,A0 +blackmail 01#A1,A0 +blame 01#A1,A2,A0 +blanket 01#A1,A2,A0 +blare 01#A1,A0 +blast 01#A1,A0 02#A1,A2,A0 03#A1,A0 04#A0 +blaze 01#A1 +bleed 01#A1,A0 02#A1,A0 03#A1,A0 04#A1,A0 +blemish 01#A1,A0 +blend 01#A1,A2,A3,A0 +bless 01#A1,A2,A0 +blindfold 01#A1,A2,A0 +blink 01#A1,A0 +block 01#A1,A2,A3,A0 +bloody 01#A1,A0 +bloom 01#A0 +blossom 01#A1 +blot 01#A1,A0 +blow 01#A1,Am,A0 02#A1,A2,A0 03#A1,A0 04#A1,Am 05#A1,A2,A3,A4,A0 06#A1,A2,A0 07#A1,A0 08#A1 09#A1,A0 10#A1,A0 +bludgeon 01#A1,A2,A0 +blunder 01#A1,A0 02#A1,A0 +blunt 01#A1,A0 +blur 01#A1,A0 +blurt 01#A1,A2,A0 +board 01#A1,A0 +boast 01#A1,A2,A0 +boat 01#A0 +bode 01#A1,A2,A0 +bog 01#A1,A0 +boil 01#A1,A0 02#A1 +bolster 01#A1,A2,A0 +bolt 01#Am,A0 02#A1,A0 +bomb 01#A1,A0 02#A1,A2 +bombard 01#A1,A0 +bond 01#A1,A2,A3,A0 02#A1 +bone 01#A1,A0 02#A1,A0 +boo 01#A1,A2,A0 +book 01#A1,A2,A0 +boom 01#A1,A2,A0 02#A0 03#A1,A0 +boost 01#A1,A2,A3,A4,Am,A0 +bootleg 01#A1,A0 +booze 01#A0 +border 01#A1,A2 +bore 01#A1,A2,A3,A0 02#A1,A2,A0 +borrow 01#A1,A2,A3,A0 +botch 01#A1,A0 +bother 01#A1,A2,A0 02#A1,A0 +bottle 01#A1,A0 02#A1,A0 +bottom 01#A0 +bounce 01#A1,A2,A3,A4,A0 02#A1 +bound 01#A0 02#A1,A0 +bow 01#A1,A0 02#A1 03#A1 04#A1,A0 06#A1,A0 +bowl 01#A0 02#A1,A0 +boycott 01#A1,A0 +brace 01#A1,A2,A0 +brag 01#A1,A0 +brake 01#A1,A0 +branch 01#A1,A0 02#A1,A0 +brand 01#A1,A2,A0 +brandish 01#A1,A0 +brave 01#A1,A0 +brazen 01#A1,A0 +breach 01#A1,A0 +bread 01#A1,A2,A0 +break 01#A1,A2,A3,A0 02#A1,A0 03#A1,A0 04#A1,A2,A0 05#A1,A0 06#A1,A0 07#A1 08#A1,A2,A0 09#A1 10#A1,A0 +breathe 01#A1,A2,A0 +breed 01#A1,A0 +brew 01#A1,A2,A3,A0 +bribe 01#A1,A2,A3,A0 +bridge 01#A1,A2,A0 +brief 01#A1,A2,A0 +brighten 01#A1,A0 +brim 01#A1,A0 +bring 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A4,AM,A0 03#A1,A2,A3,A4,AM,A0 04#A1,A2,A3,A4,A0 05#A1,A0 +bristle 01#A1,A0 +broadcast 01#A1,A2,A3,A0 +broaden 01#A1,A2,A3,A4,A0 +broker 01#A1,A2,A0 +browbeat 01#A1,A2,A0 +bruise 01#A1,A0 +brush 01#A1,A0 02#A1,A0 03#A1,A0 05#A1,A0 +buck 01#A1,A0 +buckle 01#A1,A2,A0 02#A0 03#A0 04#A1,A2 +bud 01#A1 +budge 01#A1,A2,A0 +budget 01#A1,A2,A0 +buff 01#A1,A0 +buffer 01#A1,A2,A0 +buffet 01#A1,A2,A0 +build 01#A1,A2,A4,A0 02#A1,A2,A3,A4,A0 03#A1,A2,A0 05#A1,A2,A3,A4,A0 +bulge 01#A1,A2,A0 +bulldoze 01#A1,A0 +bully 01#A1,A2,A0 +bumble 01#A1,A0 +bump 01#A1,A2,A0 02#A1,A2,A3,A4,A5,A0 03#A1,A2 +bundle 01#A1,A2,A0 02#A1,A2,A0 +bungle 01#A1,A0 +buoy 01#A1,A0 +burble 01#A1,A0 +burden 01#A1,A2,A0 +burgeon 01#A1 +burglarize 01#A1,A0 +burn 01#A1,A0 02#A1,A0 03#A1 04#A1,A2 +burnish 01#A1,A0 +burst 01#A1,A2 02#A1,A0 03#A1,A2,A3,A4,A5 +bury 01#A1,A2,A0 +bust 01#A1,A2,A3,A0 02#A1,A0 03#A1,A0 04#A1,A0 05#A1,A0 +butt 01#A0 +buttress 01#A1,A0 +buy 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A4,A0 03#A1,A2,A3,A4,A0 +buzz 01#A1,A0 02#A1,A0 03#A0 04#A1,A0 +bypass 01#A1,A0 +cadge 01#A1,A0 +cake 01#A1,A2,A0 +calculate 01#A1,A2,A0 +calibrate 01#A1,A0 +call 11#A1,A0 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 04#A1,A2,A0 05#A1,A2,A0 06#A1,A0 07#A1,A2,A0 08#A1,A2,A0 09#A1,A2,A0 10#A1,A0 +calm 01#A1,A2,A0 02#A1,A2,A0 +camouflage 01#A1,A2,A0 +camp 01#A0 +campaign 01#A1,A0 +can 01#A1,A0 02#A1,A2,A0 +cancel 01#A1,A0 +canvass 01#A1,A0 +cap 01#A1,A2,A0 02#A1,A2,A0 +capitalize 01#A1,A2,A0 02#A1,A0 +capitulate 01#A0 +captain 01#A1,A0 +caption 01#A1,A2,A0 +captivate 01#A1,A0 +capture 01#A1,A0 +care 01#A1,A0 +careen 01#A1,A0 +caress 01#A1,A0 +caricature 01#A1,A2,A0 +carp 01#A1,A0 +carpet 01#A1,A0 +carry 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A0 03#A1,A0 04#A1,A2,A3,A0 05#A1,A0 +cart 01#A1,A0 02#A1,A2,A0 +carve 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A3,A4,A0 +cascade 01#A1 +cash 01#A1,A0 02#A1,A0 03#A1,A0 +cast 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 +castigate 01#A1,A2,A0 +castle 01#A0 +catalog 01#A1,A0 +catalyze 01#A1,A0 +catapult 01#A1,A2,A0 +catch 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2 05#A1,A2 06#A1,A0 +categorize 01#A1,A2,A0 +cater 01#A1,A2,A0 +cause 01#A1,A0 +caution 01#A1,A2,A0 +cave 01#A1 02#A1 +cease 01#A1,A2,A0 +cede 01#A1,A2,A0 +celebrate 01#A1,A2,A0 02#A1,A0 +cement 01#A1,A0 +censor 01#A1,A0 +center 01#A1,A2,A0 +centralize 01#A1,A0 +certify 01#A1,A2,A0 +chafe 01#A1,A0 +chain 01#A1,A2,A3,A0 +chair 01#A1,A0 +chalk 01#A1,A0 +challenge 01#A1,A2,A0 +champion 01#A1,A0 +chance 01#A1,A0 +change 01#A1,A2,A3,A0 02#A1,A2 +channel 01#A1,A2,A3,A0 +chant 01#A1,A0 +char 01#A1,A0 +characterize 01#A1,A2,A0 +charge 01#A1,A2,A3,A0 02#A1,A0 03#A1,A2,A0 04#A1,A2,A0 05#A1,A2,A0 06#A1,A0 07#A1,A0 +chart 01#A1,A0 02#A1,A0 +charter 01#A1,A2,A0 +chase 01#A1,A0 +chasten 01#A1,A0 +chastise 01#A1,A2,A0 +chat 01#A1,A2,A0 +chauffeur 01#A1,A0 +cheapen 01#A1,A2,A0 +cheat 01#A1,A2,A0 02#A1,A0 +check 01#A1,A2,A0 02#AA,A1,A0 03#A1,A0 04#A1,A0 +cheer 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A0 +cherish 01#A1,A0 +chew 01#A1,A0 02#A1,A0 +chide 01#A1,A2,A0 +chill 01#A1,A0 +chime 01#A0 02#A1,A2,A0 +chip 01#A1,A2,A3,A0 +chisel 01#A1,A2,A4,A0 +choke 01#A1,A2,A0 02#A1,A0 03#A1,A0 04#A1,A0 +choose 01#A1,A2,A3,A4,A0 +chop 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +chortle 01#A0 02#A1,A0 +chronicle 01#A1,A0 +chuck 01#A1,A0 +chuckle 01#A0 02#A1,A2,A0 +chug 01#A0 02#A1,A0 03#A1,A2 +churn 01#A1,A2,A0 +cinch 01#A1,A0 02#A1,A2,A0 03#A1,A0 +circle 01#A0 02#A1,A2,A0 +circulate 01#A1,A0 +circumvent 01#A1,A0 +cite 01#A1,A2,A0 02#A1,A2,A0 +claim 01#A1,A2,A0 02#A1,A2,A0 +clamber 01#A0 +clamor 01#A1,A0 +clamp 01#A1,A0 +clang 01#A1,A0 +clank 01#A1 02#A1,A0 +clarify 01#A1,A2,A0 +clash 01#A1,A2,A0 +class 01#A1,A2,A0 +classify 01#A1,A2,A0 02#A1,A0 +clean 01#A1,A2,A0 02#A1,A0 03#A1,A0 +cleanse 01#A1,A2,A0 +clear 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 04#A1,A0 +click 01#A1,A0 +climb 01#A1,A0 02#A1,A2,A3,A4,AM +clinch 01#A1,A0 +cling 01#A1,A0 02#A1 +clip 01#A1,A2,A0 02#A1,A2,A0 +cloak 01#A1,A2,A0 +clobber 01#A1,A2,A0 +clock 01#A1,A2,A3,A0 02#A1,A2,A0 +clog 01#A0 02#A1,A2,A0 03#A1,A2,A0 +clone 01#A1,A0 +close 01#A1,A2,A0 02#A1,AM 03#A1,A0 04#A1,A0 05#A1,A2,A0 06#A1,A2 07#A1,A2 +clothe 01#A1,A2,A0 +cloud 01#A1,A2,A0 +club 01#A1,A0 +cluster 01#A1,A2,A0 +clutch 01#A1,A0 02#A1,A0 +clutter 01#A1,A2,A0 +co-author 01#A1,A2,A0 +co-sponsor 01#A1,A0 +coach 01#A1,A2,A0 +coast 01#A1,A0 +coat 01#A1,A2,A0 +coax 01#A1,A2,A0 +cobble 01#A1,A2,A0 +coddle 01#A1,A0 02#A1,A2,A0 +code 01#A1,A0 02#A1,A0 +codify 01#A1,A2,A0 +coerce 01#A1,A2,A0 +cohere 01#A1,A0 +coin 01#A1,A0 +coincide 01#A1 +collaborate 01#A1,A2,A0 +collapse 01#A1,A0 02#A1,A2,A3,A4 +collect 01#A1,A2,A4,A0 +color 01#A1,A2,A0 +comb 01#A1,A2,A0 02#A1,A0 +combat 01#A1,A2,A0 +combine 01#A1,A2,A3,A0 +come 11#A1,A0 01#A1,A2,A3,A4,AM 12#A1 02#A1,A0 13#A1,A2 03#A1,A2 14#A1,A2,A0 04#A1,A2 15#A1,A2 05#A1,A2,A3 16#A1,A2 06#A1,A2 17#A1,A0 07#A1,A2 18#A1 08#A1,A0 19#A1 09#A1,A2 20#A1,A0 10#A1,A0 +comfort 01#A1,A2,A0 +command 01#A1,A0 02#A1,A2,A0 +commemorate 01#A1,A2,A0 +commence 01#A1,A2,A0 +commend 01#A1,A2,A0 +comment 01#A1,A0 02#A1,A2,A3,A0 +commercialize 01#A1,A2,A0 +commission 01#A1,A2,A0 +commit 01#A1,A2,A0 02#A1,A0 +communicate 01#A1,A2,A0 +commute 01#A1,A2,A0 02#A1,A2,A0 +compact 01#A1,A0 +compare 01#A1,A2,A0 +compel 01#A1,A2,A0 +compensate 01#A1,A2,A3,A0 +compete 01#A1,A2,A0 +compile 01#A1,A4,A0 +complain 01#A1,A2,A0 +complement 01#A1,A2 +complete 01#A1,A0 +complicate 01#A1,A2,A0 +compliment 01#A1,A2,A0 +comply 01#A1,A0 +compose 01#A1,A2,A0 02#A1,A2,A0 +compound 01#A1,A0 +comprehend 01#A1,A0 +compress 01#A1,A2,A0 +comprise 01#A1,A2 +compromise 01#A1,A2,A0 02#A1,A0 +compute 01#A1,A2,A0 +computerize 01#A1,A0 +con 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 +conceal 01#A1,A2,A0 +concede 01#A1,A0 +conceive 01#A1,A2,A0 02#A0 +concentrate 01#A1,A2,A0 02#A1,A0 +concern 01#A1,A2,A0 02#A1,A0 +conclude 01#A1,A0 02#A1,A2,AM,A0 +concoct 01#A1,A2,A3,A4,A0 +concur 01#A1,A0 +condemn 01#A1,A2,A3,A0 +condition 01#A1,A2,A0 02#A1,A2,A0 +condone 01#A1,A0 +conduct 01#A1,A0 02#A1,A0 +confer 01#A1,A0 02#A1,A2,A0 +confess 01#A1,A2,A0 +confide 01#A1,A2,A0 +confine 01#A1,A2,A0 +confirm 01#A1,A2,A0 +confiscate 01#A1,A2,A0 +conflict 01#A1,A0 +conform 01#A1,A2,A0 +confront 01#A1,A2,A0 +confuse 01#A1,A2,A0 +congratulate 01#A1,A2,A0 +conjure 01#A1,A0 +connect 01#A1,A2,A0 +connote 01#A1,A0 +conquer 01#A1,A0 +consent 01#A1,A0 +conserve 01#A1,A0 +consider 01#A1,A2,A0 02#A1,A0 +consign 01#A1,A2,A0 +consist 01#A1,A2 +console 01#A1,A2,A0 +consolidate 01#A1,A2,A3,A0 +consort 01#A1,A0 +conspire 01#A1,A2,A0 +constitute 01#A1,A0 +constrain 01#A1,A2,A3,A0 +constrict 01#A1,A2,A3,A4,A0 +construct 01#A1,A2,A4,A0 +construe 01#A1,A0 +consult 01#A1,A2,A0 +consume 01#A1,A0 +consummate 01#A1,A0 +contact 01#A1,A0 +contain 01#A1,A0 +contaminate 01#A1,A2,A0 +contemplate 01#A1,A2,A0 +contemporize 01#A1,A0 +contend 01#A1,A2,A0 02#A1,A0 +content 01#A1,A2,A0 +contest 01#A1,A0 +continue 01#A1,A0 02#A1,A2,A3,A0 +contract 01#A1,A2,A3,A4 02#A1,A2,A0 03#A1,A0 04#A1,A2,A3 +contradict 01#A1,A0 +contrast 01#A1,A2,A0 +contravene 01#A1,A0 +contribute 01#A1,A2,A0 +control 01#A1,A2,A0 +convene 01#A1,A0 +converge 01#A1,A0 +convert 01#A1,A2,A3,A0 +convey 01#A1,A2,A0 +convict 01#A1,A2,A0 +convince 01#A1,A2,A0 +convolute 01#A1 +cook 01#A1,A0 02#A1,A2 03#A1,A0 +cool 01#A1,A0 02#A1,A0 03#A1,A3,A0 +cooperate 01#A1,A2,A0 +coordinate 01#A1,A2,A0 +cope 01#A1,A0 +copy 01#A1,A2,A0 02#A1,A0 +copyright 01#A1,A0 +corner 01#A1,A0 +corral 01#A1,A0 +correct 01#A1,A0 +correspond 01#A1,A0 02#A1,A2 +corroborate 01#A1,A0 +cost 01#A1,A2,A3 +costume 01#A1,A2,A0 +couch 01#A1,A2,A0 +cough 01#A0 02#A1,A2,A0 03#A1,A0 +counsel 01#A1,A2,A0 +count 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1 +countenance 01#A1,A0 +counter 01#A1,A2,A0 +counteract 01#A1,A3,A0 +counterattack 01#A1,A2,A0 +counterbalance 01#A1,A0 +countersue 01#A1,A2,A3,A0 +countervail 01#A1,A0 +couple 01#A1,A2,A0 +course 01#A1,A0 +court 01#A1,A0 +cover 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 04#A1,A0 +covet 01#A1,A0 +cower 01#A0 +crack 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 +crackle 01#A0 +craft 01#A1,A0 +cram 01#A1,A2,A0 02#A1,A0 +crane 01#A1,A0 +crank 01#A1,A0 02#A1,A0 +crash 01#A1,A0 02#A1,A0 03#A1,A0 +crave 01#A1,A0 +crawl 01#A1,A0 +creak 01#A0 +cream 01#A1,A0 02#A1,A0 +create 01#A1,A2,A3,A4,A0 +credit 01#A1,A2,A0 02#A1,A2,A0 +creep 01#A1,A0 02#A1,A0 +crest 01#A1,A0 +criminalize 01#A1,A0 +crimp 01#A1,A0 +cringe 01#A0 +cripple 01#A1,A2,A0 +crisscross 01#A1,A0 +criticize 01#A1,A2,A0 +croak 01#A1 +croon 01#A1,A0 +crop 01#A1,A2,A3,A4,A0 02#A1 +cross 01#A1,A0 02#A1,A0 03#A1,A0 04#A1,A2,A0 05#A1,A0 +crouch 01#A0 +crow 01#A1,A0 +crowd 01#A1,A2,A0 02#A1,A0 +crown 01#A1,A2,A0 +cruise 01#A0 02#A1,A0 +crumble 01#A1,A0 +crumple 01#A1,A0 +crunch 01#A1,A2,A0 +crush 01#A1,A2,A3,A0 +cry 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 +cuff 01#A1,A0 +culminate 01#A1,A2 +cultivate 01#A1,A0 +curb 01#A1,A2,A3,A4,AM,A0 +cure 01#A1,A2,A3,A0 +curl 01#A1,A2,A0 02#A1 +curry 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 +curse 01#A1,A2,A0 +curtail 01#A1,A0 +cushion 01#A1,A2,A0 +customize 01#A1,A2,A0 +cut 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A4,A0 03#A1,A0 04#A1,A2,A3,A4,A0 05#A1,A2,A3,A4,A0 06#A1,A2,A0 07#A1,A2,A0 08#A1,A0 09#A1,A2 +cycle 01#A0 +dab 01#A1,A2,A0 +dabble 01#A1,A0 +damage 01#A1,A2,A0 +damn 01#A1,A2,A0 +damp 01#A1,A0 +dampen 01#A1,A2,A0 02#A1,A0 +dance 01#A1,A2,A0 02#A1,A0 +dangle 01#A1,A0 +dare 01#A1,A2,A0 +darken 01#A1,A0 +dash 01#A1,A0 02#A1,A0 +date 01#A1,A2,A0 02#A1,A0 +daunt 01#A1,A0 +dawdle 01#A0 +dawn 01#A1,A2 02#A1 +dazzle 01#A1,A2,A0 +deactivate 01#A1,A0 +deafen 01#A1,A0 +deal 01#A1,A0 02#A1,A2,A0 +debate 01#A1,A2,A0 +debunk 01#A1,A0 +debut 01#A1,A0 +decease 01#A1 +deceive 01#A1,A2,A0 +decelerate 01#A1,A0 +decentralize 01#A1,A0 +decide 01#A1,A2,A3,A0 +decimate 01#A1,A2,A0 +deck 01#A1,A2,A0 02#A1,A2,A0 +declare 01#A1,A2,A0 02#A1,A2,A3,A0 +declassify 01#A1,A0 +decline 01#A1,A2,A3,A4,AM 02#A1,A0 +decontaminate 01#A1,A0 +decorate 01#A1,A2,A0 +decrease 01#A1,A2,A3,A4,A0 +decree 01#A1,A0 +decry 01#A1,A2,A0 +dedicate 01#A1,A2,A0 +deduct 01#A1,A2,A0 +deem 01#A1,A2,A0 +deemphasize 01#A1,A0 +deepen 01#A1,A0 +default 01#A1,A2 +defeat 01#A1,A0 +defect 01#A1,A2,A0 +defend 01#A1,A2,A0 +defer 01#A1,A2,A0 +define 01#A1,A2,A0 +deflate 01#A1,A0 +deflect 01#A1,A0 +deform 01#A1,A0 +defraud 01#A1,A2,A3,A0 +defuse 01#A1,A0 +defy 01#A1,A0 +degenerate 01#A1,A2,A3,A4,A0 +delay 01#A1,A2,A0 +delegate 01#A1,A2,A0 +delete 01#A1,A2,A0 +deliberate 01#A0 +delight 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 +delist 01#A1,A2,A0 +deliver 01#A1,A2,A0 +delouse 01#A1,A0 +delve 01#A1,A0 +demand 01#A1,A2,A0 +demean 01#A1,A0 +demilitarize 01#A2,A0 +demobilize 01#A1,A0 +democratize 01#A1,A0 +demolish 01#A1,A2,A0 +demonize 01#A1,A0 +demonstrate 01#A1,A2,A0 +demote 01#A1,A2,A3,A0 +demur 01#A1,A0 +denationalize 01#A1,A0 +denominate 01#A1,A2,A0 +denounce 01#A1,A2,A0 +denude 01#A1,A2,A0 +deny 01#A1,A2,A0 +depart 01#A1,A0 +depend 01#A1,A0 +depict 01#A1,A2,A0 +deplete 01#A1,A2,A0 +deplore 01#A1,A0 +deploy 01#A1,A2,A0 +deport 01#A1,A2,A0 +depose 01#A1,A0 +deposit 01#A1,A2,A0 +depreciate 01#A1 +depress 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +deprive 01#A1,A2,A0 +derail 01#A1,A0 +deregulate 01#A1,A0 +deride 01#A1,A2,A0 +derive 01#A1,A2,A3,A4,A0 +descend 01#A1,A0 +describe 01#A1,A2,A0 +desert 01#A1,A0 +deserve 01#A1,A2,A0 +design 01#A1,A2,A3,A0 +designate 01#A1,A2,A0 +desire 01#A1,A0 +desist 01#A0 +despair 01#A1,A0 +despise 01#A1,A0 +destabilize 01#A1,A0 +destine 01#A1,A2 +destroy 01#A1,A2,A0 +detach 01#A1,A2,A0 +detail 01#A1,A0 +detain 01#A1,A2,A0 +detect 01#A1,A2,A0 +deter 01#A1,A2,A3,A0 +deteriorate 01#A1,A2,A0 +determine 01#A1,A2,A3,A0 +detest 01#A1,A0 +dethrone 01#A1,A2,A0 +detract 01#A1,A2,A0 +devalue 01#A1,A0 +devastate 01#A1,A2,A0 +develop 01#A1,A2 02#A1,A2,A3,A4,A0 +deviate 01#A1,A0 +devise 01#A1,A2,A0 +devote 01#A1,A2,A0 +devour 01#A1,A0 +diagnose 01#A1,A2,A0 +diagram 01#A1,A0 +dial 01#A1,A0 +dicker 01#A1,A2,A0 +dictate 01#A1,A0 +die 01#A1,A2 02#A1 +differ 01#A1,A2,A0 02#A1,A2 +differentiate 01#A1,A2,A0 +dig 01#A1,A2,A0 02#A1,A0 03#A1,A0 +digest 01#A1,A0 +dignify 01#A1,A2,A0 +dilute 01#A1,A2,A3,A4,A5,A0 +diminish 01#A1,A2,A3,A4,A0 +din 01#A0 02#A0 +dine 01#A0 02#A0 +dip 01#A1,A2,A3,A4 02#A1,A2,A0 +direct 01#A1,A2,A0 +disable 01#A1,A0 +disagree 01#A1,A2,A0 +disallow 01#A1,A0 +disappear 01#A1,A0 +disappoint 01#A1,A2,A0 +disapprove 01#A1,A0 +disarm 01#A1,A0 +disassemble 01#A1,A0 +disassociate 01#A1,A2,A0 +disavow 01#A1,A0 +disband 01#A1,A0 +disburse 01#A1,A2,A3,A0 +discard 01#A1,A2,A0 +discern 01#A1,A0 +discharge 01#A1,A0 +discipline 01#A1,A2,A0 +disclaim 01#A1,A0 +disclose 01#A1,A2,A3,A0 +discolor 01#A1,A2,A0 +discomfit 01#A1,A0 +disconnect 01#A1,A2,A0 +discontinue 01#A1,A0 +discount 01#A1,A2,A3,A4,A0 02#A1,A2,A0 +discourage 01#A1,A2,A0 +discover 01#A1,A2,A3,A0 +discredit 01#A1,A0 +discriminate 01#A1,A0 02#A1,A0 +discuss 01#A1,A2,A0 +disdain 01#A1,A0 +disembark 01#A1,A0 +disenchant 01#A1,A2,A0 +disengage 01#A1,A2,A0 +disgorge 01#A1,A2,A0 +disgruntle 01#A1,A0 +disguise 01#A1,A2,A0 +disgust 01#A1,A0 +dish 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 +disincline 01#A1,A0 +disintegrate 01#A1,A0 +dislike 01#A1,A0 +dismantle 01#A1,A0 +dismay 01#A1,A2,A0 +dismember 01#A1,A0 +dismiss 01#A1,A2,A0 02#A1,A2,A0 +disobey 01#A1,A0 +disparage 01#A1,A0 +dispatch 01#A1,A2,A0 +dispel 01#A1,A0 +dispense 01#A1,A0 02#A1,A2,A0 +disperse 01#A1,A0 +displace 01#A1,A2,A0 +display 01#A1,A2,A0 +displease 01#A1,A0 +dispose 01#A1,A2,A0 +disprove 01#A1,A0 +dispute 01#A1,A2,A0 +disqualify 01#A1,A2,A0 +disregard 01#A1,A0 +disrupt 01#A1,A0 +dissatisfy 01#A1,A0 +dissect 01#A1,A0 +disseminate 01#A1,A0 +dissent 01#A1,A0 +dissipate 01#A1,A0 +dissociate 01#A1,A2,A0 +dissolve 01#A1,A2,A3,A0 +dissuade 01#A1,A2,A0 +distance 01#A1,A2,A0 +distill 01#A1,A2,A0 +distinguish 01#A1,A2,A0 +distort 01#A1,A0 +distract 01#A1,A2,A0 +distribute 01#A1,A2,A0 +disturb 01#A1,A2,A0 +dither 01#A0 +dive 01#A1,A2,A0 02#A1,A2,A3,A4 +diverge 01#A0 +diversify 01#AA,A1,A2,A0 +divert 01#A1,A2,A3,A0 +divest 01#A1,A2,A0 +divide 01#A1,A2,A0 02#A1,A2,A0 +divorce 01#A1,A0 02#A1,A2,A0 +divvy 01#A1,A2,A0 +dizzy 01#A1,A0 +do 02#A1,A2,A3,A4,A0 03#A1,A0 +doctor 01#A1,A0 +document 01#A1,A0 +dodge 01#A1,A0 +dog 01#A1,A0 +dole 01#A1,A2,A0 +dominate 01#A1,A0 +don 01#A1,A0 +donate 01#A1,A2,A0 +doom 01#A1,A2,A0 +dot 01#A1,A2,A0 +double 01#A1,A2,A3,A4,A0 02#A1,A0 +double-cross 01#A1,A0 +doublecross 01#A1,A0 +doubt 01#A1,A0 +dovetail 01#A1,A2 +down 01#A1,A0 02#A1,A0 +downgrade 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +draft 01#A1,A2,A3,A0 +drag 01#A1,A2,A0 +drain 01#A1,A2,A0 +drape 01#A1,A2,A0 +draw 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1 05#A1,A2,A0 +dread 01#A1,A0 +dream 01#A1,A0 02#A1,A0 03#A0 +dress 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 +drift 01#A1 +drill 01#A1,A2,A3,A4,A0 02#A1,A2,A0 +drink 01#A1,A2,A0 02#A1,A0 +drip 01#A1,A2,A3,A0 02#A1,A2 +drive 01#A1,A0 02#A1,A2,A0 +drool 01#A1,A2,A0 02#A1,A0 +drop 01#A1,A2,A3,A4,AM,A0 02#A1,A2,A3,A4 03#A1,A2,A4,AM,A0 04#A1,A0 05#A1,A2,A0 06#A1,A0 07#A1,A2,A0 +drown 01#A1,A0 02#A1,A0 03#A1,A0 +drum 01#A1,A0 +dry 01#A1,A0 +dub 01#A1,A2,A0 +duck 01#A1,A0 +dull 01#A1,A0 +dump 01#A1,A2,A0 +dupe 01#A1,A2,A0 +duplicate 01#A1,A0 +dust 01#A1,A2,A0 02#A1,A0 03#A1,A0 +dwarf 01#A1,A0 +dwindle 01#A1,A2,A3,A4 +dye 01#A1,A2,A0 +earmark 01#A1,A2,A0 +earn 01#A1,A2,A3,A0 +ease 01#A1,A2,A3,A4,AM 02#A1,A0 03#A1,A3,A4,A0 04#A1,A0 +eat 01#A1,A0 02#A1,A0 03#A1,A0 +eavesdrop 01#A1,A0 +ebb 01#A1 +echo 01#A1,A0 +eclipse 01#A1,A0 +economize 01#A1,A0 +edge 01#A1,A2,A3,A4,A5,AM 02#A1,A2,A0 +edit 01#A1,A0 +educate 01#A1,A2,A0 +effect 01#A1,A0 +eject 01#A1,A2,A0 +elaborate 01#A1,A0 +elapse 01#A1 +elect 01#A1,A2,A0 +electrify 01#A1,A0 +elevate 01#A1,A2,A0 +eliminate 01#A1,A2,A0 +elongate 01#A1,A0 +elude 01#A1,A0 +emasculate 01#A1,A0 +embargo 01#A1,A0 +embark 01#A1,A0 +embarrass 01#A1,A2,A0 +embed 01#A1,A2,A0 +embellish 01#A1,A2,A0 +embezzle 01#A1,A2,A0 +embody 01#A1,A2 +embolden 01#A1,A2,A0 +embrace 01#A1,A2,A0 02#A1,A0 +embroil 01#A1,A2,A0 +emerge 01#A1,A0 02#A1,A0 +emigrate 01#A1,A2,A0 +emit 01#A1,A0 +emote 01#A0 +empathize 01#A1,A0 +emphasize 01#A1,A2,A0 +employ 01#A1,A2,A0 02#A1,A2,A0 +empower 01#A1,A2,A0 +empty 01#A1,A2,A0 +emulate 01#A1,A0 +enable 01#A1,A0 +enact 01#A1,A2,A0 +encapsulate 01#A1,A0 +encase 01#A1,A2,A0 +encircle 01#A1,A2,A0 +enclose 01#A1,A2,A0 +encompass 01#A1,A0 +encounter 01#A1,A0 +encourage 01#A1,A2,A0 02#A1,A2,A0 +encroach 01#A1,A0 +encrust 01#A1,A2,A0 +encrypt 01#A1,A0 +encumber 01#A1,A2,A0 +end 01#A1,A2,AM,A0 02#A1,A2,A3,A4 03#A1,A2 +endanger 01#A1,A0 +endeavor 01#A1,A0 +endorse 01#A1,A2,A0 02#A1,A0 +endow 01#A1,A2,A0 +endure 01#A1,A2 +energize 01#A1,A2,A0 +enforce 01#A1,A0 +engage 01#A1,A2,A0 +engineer 01#A1,A4,A0 +engrave 01#A1,A2,A0 +engulf 01#A1,A0 +enhance 01#A1,A2,A3,A0 +enjoin 01#A1,A2,A3,A0 +enjoy 01#A1,A0 +enlarge 01#A1,A2,A3,A4,A0 +enlighten 01#A1,A2,A3,A0 +enlist 01#A1,A2,A0 +enliven 01#A1,A0 +enrage 01#A1,A2,A0 +enrich 01#A1,A2,A0 +enroll 01#A1,A2,A0 +ensconce 01#A1,A2,A0 +ensnarl 01#A1,A2,A0 +ensue 01#A1,A2 +ensure 01#A1,A2,A0 +entail 01#A1,A0 +entangle 01#A1,A2,A0 +enter 01#A1,A0 02#A1,A2,A3,A0 +entertain 01#A1,A2,A0 02#A1,A0 +entice 01#A1,A2,A3,A0 +entitle 01#A1,A2,A0 02#A1,A2,A0 +entomb 01#A1,A2,A0 +entrance 01#A1,A2,A0 +entrench 01#A1,A2,A0 +entrust 01#A1,A2,A0 +entwine 01#A1,A2,A0 +envisage 01#A1,A2,A0 +envision 01#A1,A0 +envy 01#A1,A2,A0 +epitomize 01#A1,A0 +equal 01#A1,A2 +equate 01#A1,A2,A0 +equip 01#A1,A2,A0 +eradicate 01#A1,A0 +erase 01#A1,A2,A3,A0 +erect 01#A1,A0 +erode 01#A1,A0 +err 01#A0 +erupt 01#A1,A2 02#A1,A2,A0 +escalate 01#A1,A0 +escape 01#A1,A0 +eschew 01#A1,A0 +escort 01#A1,A3,A4,A0 +espouse 01#A1,A0 +establish 01#A1,A2,A3,A0 +estimate 01#A1,A2,A0 +estrange 01#A1,A2 +evacuate 01#A1,A2,A0 +evade 01#A1,A0 +evaluate 01#A1,A0 +evaporate 01#A1,A0 +even 01#A1,A0 +evidence 01#A1,A0 +evince 01#A1,A0 +eviscerate 01#A1,A0 +evoke 01#A1,A2,A0 +evolve 01#A1,A2,A3 02#A1,A0 +exacerbate 01#A1,A0 +exaggerate 01#A1,A0 +examine 01#A1,A0 +excavate 01#A1,A0 +exceed 01#A1,A2,A0 +excel 01#A1,A0 +exchange 01#A1,A2,A3,A4,A0 +excise 01#A1,A2,A0 +excite 01#A1,A2,A0 +exclaim 01#A1,A2,A0 +exclude 01#A1,A2,A0 +excoriate 01#A1,A2,A0 +excuse 01#A1,A2,A0 +execute 01#A1,A2,A0 02#A1,A2,A3,A0 +exemplify 01#A1,A0 +exempt 01#A1,A2,A0 +exercise 01#A1,A0 02#A0 +exert 01#A1,A2,A0 +exhale 01#A1,A0 +exhaust 01#A1,A0 +exhibit 01#A1,A2,A0 +exhort 01#A1,A2,A0 +exile 01#A1,A2,A0 +exist 01#A1,A2 +exit 01#A1,A0 +exonerate 01#A1,A2,A0 +exorcise 01#A1,A2,A0 +expand 01#A1,A2,A3,A4,A0 +expect 01#A1,A0 +expedite 01#A1,A0 +expel 01#A1,A2,A0 +expend 01#A1,A2,A0 +experience 01#A1,A0 +experiment 01#A1,A2,A0 +expire 01#A1,AM +explain 01#A1,A2,A0 +explode 01#A1,A2,A0 02#A1,A2,A3,A4,A0 03#A1,A2 +exploit 01#A1,A2,A0 +explore 01#A1,A0 +export 01#A1,A2,A0 +expose 01#A1,A2,A0 +expound 01#A1,A0 +express 01#A1,A2,A0 02#A1,A2,A0 +expunge 01#A1,A2,A0 +extend 01#A1,A2,A3,A4,A0 02#A1,A2,A0 +extinguish 01#A1,A2,A0 +extort 01#A1,A2,A0 +extract 01#A1,A2,A0 +extradite 01#A1,A2,A3,A0 +extrapolate 01#A1,A2,A0 +extricate 01#A1,A2,A0 +exude 01#A1,A0 +eye 01#A1,A0 +eyeball 01#A1,A0 +fabricate 01#A1,A2,A3,A0 +face 01#A1,A0 +facilitate 01#A1,A0 +factor 01#A1,A2,A0 +fade 01#A1,A2,A3,A4,A0 +fail 01#A1,A2,A3,A0 02#A1,A0 +faint 01#A0 +fake 01#A1,A0 +fall 01#A1,A2,A3,A4,AM 02#A1,A2 03#A1 04#A1,A2 05#A1,A2 06#A1 07#A1,A2,A0 +falsify 01#A1,A0 +falter 01#A1 +familiarize 01#A1,A2,A0 +fan 01#A1,A0 +fancy 01#A1,A0 02#A1,A0 +fantasize 01#A1,A0 +fare 01#Am,A0 +farm 01#A1,A0 02#A1,A2,A0 +fascinate 01#A1,A2,A0 +fashion 01#A1,A0 +fasten 01#A1,A2,A3,A0 +fatten 01#A1,A0 +fault 01#A1,A2,A0 +favor 01#A1,A2,A0 +fawn 01#A1,A0 +fax 01#A1,A0 +fear 01#A1,A0 +feast 01#A1,A0 +feature 01#A1,A2,A0 +feed 01#A1,A2,A0 02#A1,A0 03#A1,A2 +feel 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2,A0 05#A1,A2 +fell 01#A1,A0 +fend 01#A1,A0 02#A1,A2,A0 +ferret 01#A1,A0 +ferry 01#A1,A2,A3,A0 +fertilize 01#A1,A0 +fester 01#A1 +festoon 01#A1,A2,A0 +fetch 01#A1,A2,A3,A4,A0 +feud 01#A1,A2,A0 +fidget 01#A1,A0 +field 01#A1,A0 +fight 01#A1,A2,A0 02#A1,A2,A0 +figure 01#A1,A0 02#A1,A2,A0 03#A1,A0 04#A1,A2,A3,A0 05#A1,A2,A3,A0 +filch 01#A1,A2,A0 +file 01#A1,A2,A3,A4,A0 02#A1,A0 03#A0 +fill 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 04#A1,A0 05#A1,A0 06#A1,A2,A0 +film 01#A1,A0 +filter 02#A1,A2,A0 03#A1,A2 +finagle 01#A1,A2,A0 +finalize 01#A1,A0 +finance 01#A1,A2,A0 02#A1,A2,A0 +find 01#A1,A2,A3,A0 +fine 01#A1,A2,A3,A0 +fine-tune 01#A1,A0 +finger 01#A1,A0 02#A1,A0 03#A1,A2,A0 +finish 01#A1,A2,AM,A0 02#A1,AM 03#A1,A2,A0 +fire 01#A1,A0 02#A1,A2,A0 03#A1,A0 +fireproof 01#A1,A0 +firm 01#A1,A2,A3,A4,A0 02#A1,A0 +fish 01#A1,A0 +fit 01#A1,A0 +fix 01#A1,A3,A0 02#A1,A2,A0 03#A1,A2,A0 +fizz 01#A0 +fizzle 01#A1 +flabbergast 01#A1,A0 +flame 01#A1 +flank 01#A1,A0 +flap 01#A1,A0 +flare 01#A1 +flash 01#A1,A2,A0 02#A1,A0 03#A1,A2 +flatten 01#A1,A0 02#A1,A0 +flaunt 01#A1,A0 +flaw 01#A1,A0 +flay 01#A1,A2,A0 +flee 01#A1,A0 +fleece 01#A1,A2,A0 +flicker 01#A1 +flinch 01#A0 +fling 01#A1,A2,A0 +flip 01#A1,A2,A0 +flirt 01#A1,A0 +float 01#A1,A2,A0 +flock 01#A0 02#A1,A0 +flog 01#A1,A0 +flood 01#A1,A2,A0 +flop 01#A1,A0 +flounder 01#A0 +flourish 01#A1,A2 +flout 01#A1,A0 +flow 01#A1,Am +fluctuate 01#A1,A2,A3,A4,A5 +flunk 01#A1,A2,A0 +flush 01#A1,A2,A0 02#A1,A0 +fly 01#A1,A2,A0 +foam 01#A1,A0 +focus 01#A1,A2,A0 +fog 01#A1,A0 +foil 01#A1,A2,A0 +fold 01#A1,A2,A0 02#A1,A0 03#A1,A2,A3,A4,A0 04#A0 05#A1,A2,A3,A4,A0 06#A1,A0 +follow 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1 05#A1,A2 06#A0 07#A1,A0 +foment 01#A1,A0 +fool 01#A1,A0 02#A0 +foot 01#A1,A0 +forbid 01#A1,A2,A0 +force 01#A1,A2,A0 02#A1,A2,A0 +forecast 01#A1,A0 +foreclose 01#A1,A0 +foresee 01#A1,A0 +foreshadow 01#A1,A2,A0 +forfeit 01#A1,A2,A0 +forge 01#A1,A0 02#A1,A2,A4,A0 +forget 01#A1,A0 +forgive 01#A1,A2,A0 +forgo 01#A1,A0 +fork 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +form 01#A1,A2,A3,A4,A0 02#A1,A0 +formalize 01#A1,A0 +formulate 01#A1,A2,A3,A4,A0 +forsake 01#A1,A0 +forward 01#A1,A2,A0 +foster 01#A1,A0 +foul 01#A1,A2,A0 +found 01#A1,A2,A0 +founder 01#A1,A2 +fraction 01#A1,A0 +fracture 01#A1,A0 +fragment 01#A1,A0 +frame 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 04#A1,A0 05#A1,A0 +franchise 01#A1,A2,A0 +fray 01#A1,A0 +freak 01#A1 02#A1,A0 +free 01#A1,A2,A3,A0 +freeze 01#A1,A2,A0 02#A1,A2,A3,A0 +frequent 01#A1,A0 +fret 01#A1,A0 +frighten 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A2,A0 +fritter 01#A1,A0 +frolic 01#A0 +froth 01#A1 +frown 01#A0 02#A1,A0 +frustrate 01#A1,A2,A0 +fry 01#A1,A0 +fudge 01#A1,A0 02#A1,A0 +fuel 01#A1,A2,A0 +fulfill 01#A1,A0 +fume 01#A1,A0 02#A1,A2,A0 +function 01#A1,A0 +fund 01#A1,A2,A0 +fundraise 01#A2,A0 +funnel 01#A1,A2,A3,A0 +furlough 01#A1,A0 +furnish 01#A1,A2,A0 +furrow 01#A1 +further 01#A1,A0 +fuss 01#A1,A0 +gain 01#A1,A2,A3,A4,AM 02#A1,A2,A3,A0 +gallop 01#AA,A1,A0 +galvanize 01#A1,A2,A0 02#A1,A0 03#A1,A0 +gamble 01#A1,A0 +gape 01#A1,A0 +garden 01#A0 +garner 01#A1,A0 +gasp 01#A1,A0 02#A1,A2,A0 +gather 01#A1,A2,A3,A0 +gauge 01#A1,A2,A3,A0 +gaze 01#A1,A0 +gear 01#A1,A2,A0 02#A1,A2,A0 +generalize 01#A1,A0 +generate 01#A1,A2,A3,A4,A0 +gesture 01#A1,A0 +get 11#A1,A2 01#A1,A2,A3,A4,A0 12#A1,Am,A0 02#A1,A0 13#A1,A0 03#A1,A2 14#A1,A0 04#A1,A2,A0 15#A1,A2 05#A1,A2,A0 16#A1,A2,A0 06#A1,A0 07#A1,A2,A3,A0 08#A1,A0 09#A1,A2,A0 10#A1,A2 +gird 01#A1,A2,A0 02#A1,A2,A0 +give 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A0 05#A1 06#A1,A2,A0 07#A1,A0 08#A1,A2,A0 09#A1,A0 +glamorize 01#A1,A2,A0 +glance 01#A1,A0 +glare 01#A1,A0 +glaze 01#A1,A0 02#A1 +gleam 01#A0 +glean 01#A1,A2,A0 +glide 01#A0 +glisten 01#A0 +gloat 01#A1,A0 +gloss 01#A1,A0 +glow 01#A1,A0 +glue 01#A1,A2,A0 +glut 01#A1,A2,A0 +go 11#A1,A2 22#A1,A2 01#A1,A2,A3,A4,AM 12#A1,A2 02#A1,A3,A4,AM,A0 13#A1 03#A1,A0 14#A1 15#A1 05#A1,A0 16#A1 06#A1,A2,A0 17#A1,A0 07#A1,A2,A3 18#A1 08#A1,A2,A3 19#A1,A2 09#A1 20#A1 10#A1,A2,AM 21#A1,A2 +gobble 01#A1,A0 02#A0 03#A1,A0 +gore 01#A1,A2,A0 +gossip 01#A1,A0 +govern 01#A1,A0 +grab 01#A1,A0 +grace 01#A1,A0 +grade 01#A1,A2,A0 +graduate 01#AA,A1,A2,A0 +graft 01#A1,A2,A0 +grant 01#A1,A2,A0 +grapple 01#A1,A0 +grasp 01#A1,A0 +gray 01#A1,A0 +graze 01#AA,A1,A0 +greet 01#A1,A2,A3,A0 +grimace 01#A1,A0 +grin 01#A0 02#A1,A2,A0 +grind 01#A1,A2,A0 02#A1,A2,A0 03#A1 04#A1,A2,A0 05#A1,A2,A0 +grip 01#A1,A2,A3,A0 +gripe 01#A1,A0 +groan 01#A1,A0 +grok 01#A1,A0 +grope 01#A1,A0 +gross 01#A1,A0 +ground 01#A1,A0 +group 01#A1,A2,A0 +grouse 01#A1,A2,A0 +grovel 01#A0 +grow 01#A1,A2,A3,A4,A0 02#A1,A2,A3 03#A1,A2,A0 04#A1 +growl 01#A1,A0 +grumble 01#A1,A2,A0 +grunt 02#A1,A2,A0 +guarantee 01#A1,A2,A0 +guard 01#A1,A2,A0 +guess 01#A1,A0 +guide 01#A1,A2,A0 +gun 01#A1,A0 +gush 01#A1,A0 02#A1,A0 +gut 01#A1,A0 +guzzle 01#A1,A0 +gyrate 01#A1,A0 +hack 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A3,A0 04#A1,A0 05#A0 +haggle 01#A1,A2,A0 +hail 02#A1,A2,A0 03#A1 04#A1,A0 +halt 01#A1,A2,A0 +halve 01#A1,A0 +ham 01#A1,A0 +hammer 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A3,A4,A0 04#A1,A2,A0 +hamper 01#A1,A2,A0 +hamstring 01#A1,A0 +hand 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +handcuff 01#A1,A2,A0 +handicap 01#A1,A0 +handle 01#A1,A2,A3,A0 +hang 01#A1,A2,A0 02#A1,A2 03#A1,A0 +happen 01#A1,A2 02#A1 03#A1,A0 +harass 01#A1,A2,A0 02#A1,A2,A0 +harbor 01#A1,A0 +harden 01#A1,A0 +harm 01#A1,A2,A0 +harness 01#A1,A0 +harp 01#A1,A0 +harry 01#A1,A0 +harvest 01#A1,A0 +hash 01#A1,A0 +hasten 01#A1,A0 +hatch 01#A1,A0 +hate 01#A1,A0 +haul 01#A1,A2,A3,A4,A0 02#A1,A2,A0 03#A1,A0 +haunt 01#A1,A0 +have 03#A1,A0 04#A1,A0 +hawk 01#A1,A0 02#A0 +head 01#A1,A0 02#A1,A0 03#A1,A2,A0 +headline 01#A1,A2,A0 +heal 01#A1,A0 +heap 01#A1,A2,A0 +hear 01#A1,A2,A0 +hearten 01#A1,A0 +heat 01#A1,A2,A0 02#A1,A0 +heave 01#A1,A2,A0 02#A1,A0 03#A1 +heckle 01#A1,A0 +hedge 01#A1,A2,A3,A0 +heed 01#A1,A0 +heighten 01#A1,A0 +help 01#A1,A2,A0 +hemorrhage 01#A1,A0 +herald 01#A1,A2,A0 +herd 01#A1,A2,A0 +hesitate 01#A1,A0 +hew 01#A1,A2,A0 02#A1,A0 +hide 01#A1,A2,A0 +highlight 01#A1,A2,A0 +hightail 01#A1,A2,A0 +hinder 01#A1,A2,A3,A0 +hinge 01#A1,A2 +hint 01#A1,A2,A0 +hire 01#A1,A2,A0 +hiss 01#A1,A0 +hit 01#A1,A2,A0 02#A1,A0 03#A1,A0 +hitch 01#A1,A2,A0 +hoard 01#A1,A0 +hobble 01#Am,A0 02#A1,A2,A0 +hoe 01#A1,A0 +hoist 01#A1,A2,A0 +hold 11#A1 01#A1,A2,A3,A0 12#A1,A2,A0 02#A1,A2,A0 03#A1,A2 04#A1,A2,A0 05#A1 06#A1,A2,A0 07#A1,A2,A0 08#A1,A0 09#A1,A2,Am 10#A1,A0 +hole 01#A1,A0 +holler 01#A1,A0 +homer 01#A0 +hone 01#A1,A2,A0 +honor 01#A1,A2,A0 +hoodwink 01#A1,A2,A0 +hook 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A2 +hop 01#A1,A0 +hope 01#A1,A0 +hospitalize 01#A1,A0 +host 01#A1,A0 +hound 01#A1,A0 +house 01#A1,A2,A0 +hover 01#A1,A2 +howl 01#A1,A2,A0 02#A1,A0 03#A0 +huckster 01#A1,A0 +huddle 01#A0 02#A1,A0 +hug 01#A1,A0 +hulk 01#A1 +hum 01#A0 02#A1,A0 +humanize 01#A1,A0 +humble 01#A1,A0 +hunch 01#A0 +hunt 01#A1,A0 +hurl 01#A1,A2,A0 +hurry 01#A1,A2,A0 +hurt 01#A1,A2,A3,A0 02#A1 +hurtle 01#A1,A0 +hustle 01#AA,A1,A0 +hypnotize 01#A1,A0 +hypothesize 01#A1,A0 +identify 01#A1,A2,A0 02#A1,A2,A0 +idle 01#A1,A0 +ignite 01#A1,A0 +ignore 01#A1,A0 +illuminate 01#A1,A0 +illustrate 01#A1,A2,A0 02#A1,A2,A0 +imagine 01#A1,A2,A0 +imitate 01#A1,A0 +immerse 01#A1,A2,A0 +immigrate 01#A1,A2,A0 +impact 01#A1,A0 02#A1,A2 +impair 01#A1,A0 +impart 01#A1,A2,A0 +impeach 01#A1,A2,A0 +impede 01#A1,A2,A3,A0 +impel 01#A1,A2,A0 +impend 01#A1 +imperil 01#A1,A0 +implant 01#A1,A2,A0 +implement 01#A1,A2,A0 +implicate 01#A1,A2,A0 +implore 01#A1,A2,A0 +imply 01#A1,A2,A0 +import 01#A1,A2,A0 +impose 01#A1,A2,A0 +impound 01#A1,A0 +impoverish 01#A1,A0 +impress 01#A1,A2,A0 02#A1,A2,A0 +imprison 01#A1,A2,A3,A0 +improve 01#A1,A0 02#A1,A2,A3,A4,AM +improvise 01#A1,A0 +impugn 01#A1,A0 +inaugurate 01#A1,A2,A0 02#A1,A0 +incarcerate 01#A1,A2,A3,A0 +inch 01#A1,A2,A3,A4,A5,A0 +incite 01#A1,A0 +incline 01#A1,A0 +include 01#A1,A2,A0 +incorporate 01#A1,Am,A0 02#A1,A2,A0 +increase 01#A1,A2,A3,A4,A0 +incriminate 01#A1,A2,A0 +incur 01#A1,A2,A0 +indemnify 01#A1,A2,A0 +index 01#A1,A2,A0 +indicate 01#A1,A2,A0 +indict 01#A1,A2,A0 +indoctrinate 01#A1,A2,A0 +induce 01#A1,A0 02#A1,A2,A0 +indulge 01#A1,A2,A0 +industrialize 01#A1 +infect 01#A1,A2,A0 +infer 01#A1,A2,A0 +infiltrate 01#A1,A0 +inflame 01#A1,A0 +inflate 01#A1,A2,A3,A4,A0 +inflict 01#A1,A2,A0 +influence 01#A1,A2,A0 +inform 01#A1,A2,A0 +infringe 01#A1,A0 +infuriate 01#A1,A0 +infuse 01#A1,A2,A0 +ingest 01#A1,A0 +ingratiate 01#A1,A2,A0 +inhabit 01#A1,A0 +inherit 01#A1,A2,A0 +inhibit 01#A1,A0 +initial 01#A1,A0 +initiate 01#A1,A2,A0 02#A1,A2,A3,A0 +inject 01#A1,A2,A0 +injure 01#A1,A2,A0 +innovate 01#A1,A0 +inquire 01#A1,A0 +insert 01#A1,A2,A0 +insinuate 01#A1,A0 +insist 01#A1,A0 +inspect 01#A1,A2,A0 +inspire 01#A1,A2,A0 +install 01#A1,A2,A0 +instigate 01#A1,A0 +instill 01#A1,A2,A0 +institute 01#A1,A2,A0 +instruct 01#A1,A2,A0 +insulate 01#A1,A2,A3,A0 02#A1,A2,A0 +insult 01#A1,A0 +insure 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +integrate 01#A1,A2,A0 +intend 01#A1,A2,A0 +intensify 01#A1,A2,A0 +intercede 01#A1,A2,A0 +intercept 01#A1,A0 +interconnect 01#A1,A0 +interest 01#A1,A2,A0 +interfere 01#A1,A0 +interject 01#A1,A2,A0 +interlope 01#A0 +intermix 01#A1,A2,A0 +interpret 01#A1,A2,A0 +interrogate 01#A1,A2,A0 +interrupt 01#A1,A0 +intersperse 01#A1,A2,A0 +intertwine 01#A1,A0 +intervene 01#A1,A0 +interview 01#A1,A2,A0 +intimate 01#A1,A2,A0 +intimidate 01#A1,A0 +intone 01#A1,A0 +introduce 01#A1,A2,A0 02#A1,A2,A0 +intrude 01#A1,A2,A0 +inundate 01#A1,A2,A0 +invade 01#A1,A0 +invalidate 01#A1,A0 +invent 01#A1,A0 +invert 01#A1,A0 +invest 01#A1,A2,A0 +investigate 01#A1,A2,A0 +invite 01#A1,A2,A0 +invoke 01#A1,A0 +involve 01#A1,A2,A0 +irk 01#A1,A2,A0 +iron 01#A1,A0 +irradiate 01#A1,A2,A0 +irritate 01#A1,A0 +isolate 01#A1,A0 +issue 01#A1,A2,A3,A0 +itemize 01#A1,A0 +jack 01#A1,A0 +jail 01#A1,A2,A3,A0 +jam 01#A1,A2,A0 +jar 01#A1,A0 +jell 01#A1 +jeopardize 01#A1,A0 +jerk 01#A1,A2,A0 +jet 01#A1,A2 +jettison 01#A1,A0 +jiggle 01#A1,A0 +jingle 01#A1,A0 +jinx 01#A1,A0 +jog 01#A1,A0 +join 01#A1,A2,A0 +joke 01#A1,A2,A0 +jolt 01#A1,A2,A0 02#A1,A2,A0 +jostle 01#A1,A0 +joust 01#A1,A0 +judge 01#A1,A2,A3,A0 +juggle 01#A1,A0 +jump 01#A1,A2,A3,A4 02#A1,A0 03#A0 04#A1,A0 +junk 01#A1,A0 +justify 01#A1,A2,A3,A4,A0 +jut 01#A1,A2 02#A1,A2 +juxtapose 01#A1,A2,A0 +kayo 01#A1,A2,A0 +keep 01#A1,A0 02#A1,A0 03#A1,A2,A0 04#A1,A2,A0 05#A1,A0 08#A1 +key 01#A1,A2,A0 +kick 01#A1,A2,A0 02#A1 03#A1,A2,A0 04#A1,A2,A0 +kid 01#A1,A2,A0 +kidnap 01#A1,A0 +kill 01#A1,A2,A0 +kindle 01#A1,A0 +kiss 01#A1,A2,A0 +knead 01#A1,A2,A0 +kneel 01#A0 +knit 01#A1,A2,A4,A0 02#A1 +knock 01#A1,A2,A0 02#A1,A2,A3,A4,A5,A0 03#A1,A2,A0 04#A1,A2,A3,A0 05#A1,A0 06#A1,A0 +know 01#A1,A2,A0 +kowtow 01#A1,A0 +label 01#A1,A2,A0 +labor 01#A1,A0 +lace 01#A1,A0 02#A1,A2,A0 +lack 01#A1,A0 +lag 01#A1,A2,A0 +lambaste 01#A1,A2,A0 +lament 01#A1,A0 +laminate 01#A1,A0 +land 01#A1,A0 02#A1,A2,A0 +languish 01#A1,A2 02#A1,A0 +lap 01#A1,A0 +lapse 01#A1 +lash 01#A1,A2,A0 02#A1,A0 +last 01#A1,A2 +latch 01#A1,A0 02#A1,A0 +laud 01#A1,A2,A0 +laugh 01#A1,A2,A0 02#A1,A2,A0 +launch 01#A1,A0 +launder 01#A1,A0 +lavish 01#A1,A2,A0 +lay 01#A1,A2,A0 02#A1,A0 +lead 01#A1,A3,A4,A0 02#A1,A0 03#A1,A2,A0 04#A1,A2,A3,A0 +leak 01#A1,A0 +lean 01#A1,A2,A0 +leap 01#A1,A2,A3,A4 02#A1,A0 03#A0 +leapfrog 01#A1,A0 +learn 01#A1,A2,A0 +lease 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A4,A0 +leave 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +lecture 01#A1,A2,A0 +legalize 01#A1,A0 +legislate 01#A1,A0 +legitimize 01#A1,A0 +lend 01#A1,A2,A0 +lengthen 01#A1,A2,A3,A4,A0 +lessen 01#A1,A2,A3,A4,AM,A0 +let 01#A1,A0 02#A1,A2,A0 03#A1,A0 +level 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +leverage 01#A1,A0 +levy 01#A1,A0 +libel 01#A1,A0 +liberalize 01#A1,A0 +liberate 01#A1,A2,A0 +license 01#A1,A2,A0 +lick 01#A1,A0 02#A1,A2,A0 +lie 01#A1,A2 02#A1,A2,A0 03#A1,A2 +lift 01#A1,A2,A3,A4,AM,A0 02#A1,A2,A0 +light 01#A1,A0 02#A1,A0 +lighten 01#A1,A2,A3,A4,A0 02#A1,A2,A0 03#A1,A0 +like 01#A1,A0 02#A1,A0 +liken 01#A1,A2,A0 +limit 01#A1,A2,A0 +limp 01#A0 +line 01#A1,A2,A0 02#A1,A0 +linger 01#A1,A2 +link 01#A1,A2,A0 02#A1,A2,A0 +liquidate 01#A1,A0 +liquify 01#A1,A0 +list 01#A1,A2,A0 02#A1,A2,A0 03#A1 +listen 01#A1,A0 +litter 01#A1,A2,A0 +live 01#A1,A0 02#A1,A0 03#A1,A2,A3 04#A1,A0 +load 01#A1,A2,A3,A0 02#A1,A2,A0 +loan 01#A1,A2,A0 02#A1,A0 +loathe 01#A1,A0 +lobby 01#A1,A2,A0 +localize 01#A1,A0 +locate 01#A1,AM,A0 02#A1,A0 +lock 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A0 05#A1,A2,A0 +lodge 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 +log 01#A1,A0 02#A1,A0 +log-roll 01#A0 +long 01#A1,A0 +look 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 04#A1,A0 05#A1,A2,A0 +loom 01#A1 +loosen 01#A1,A2,A0 02#A1,A2,A0 +loot 01#A1,A2,A0 +lop 01#A1,A2,A0 +lord 01#A1,A2,A0 +lose 01#A1,A2,A3,A4,AM 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2,A0 06#A1,A2,A0 07#A1,A2 +love 01#A1,A0 02#A1,A0 +lower 01#A1,A2,A3,A4,A0 +lug 01#A1,A0 +lull 01#A1,A2,A0 +lumber 01#A1,A0 +lump 01#A1,A2,A0 +lunge 01#A1,A0 +lurch 01#A1,A0 +lure 01#A1,A2,A3,A0 +lurk 01#A0 +magnetize 01#A1,A0 +magnify 01#A1,A0 +mail 01#A1,A2,A0 +maintain 01#A1,A2,A0 +major 01#A1,A0 +make 11#A1,A2 01#A1,A2,A3,A0 12#A1,A0 02#A1,A2,A0 13#A1,A2,A0 03#A1,A2,A0 14#A1,A0 04#A1,A2,A3,A0 15#A1 05#A1,A2,A0 16#A1,A2,A0 06#A1,A2 17#A1,A0 07#A1,A0 08#A1,A0 09#A1,A0 10#A1,A2,A3,A0 +malign 01#A1,A0 +man 01#A1,A0 +manage 01#A1,A0 02#A1,A0 +mandate 01#A1,A0 +maneuver 01#A1,A2,A0 +manhandle 01#A1,A0 +manifest 01#A1,A2 +manipulate 01#A1,A0 02#A1,A2,A0 +manufacture 01#A1,A2,A3,A0 +map 01#A1,A0 02#A1,A2,A0 +mar 01#A1,A2,A0 +march 01#AA,A0 02#A1 +marginalize 01#A1,A0 +mark 01#A1,A0 02#A1,A2,A3,A0 03#A1,A2,A0 04#A1,A0 05#A1,A0 +market 01#A1,A2,A3,A4,A0 +marry 01#A1,A2,A0 +marvel 01#A1,A0 02#A1,A2,A0 +mask 01#A1,A2,A0 +masquerade 01#A1,A0 +mass 01#A0 +massage 01#A1,A0 +master 01#A1,A0 +match 01#A1,A0 +mate 01#A1,A2,A0 02#AA,A1,A0 +materialize 01#A1 +matter 01#A1,A2 +mature 01#A1 +maul 01#A1,A0 +maximize 01#A1,A0 +mean 01#A1,A0 +meander 01#A1,A0 +measure 01#A1,A2,A3,A0 +meddle 01#A1,A0 +mediate 01#A1,A0 +meet 01#A1,A0 02#A1,A0 03#A1,A0 +meld 01#A1,A2,A0 +mellow 01#A1,A2 +melt 01#A1,A0 +memorialize 01#A1,A0 +memorize 01#A1,A0 +mend 01#A1,A0 +mention 01#A1,A2,A3,A0 +merchandise 01#A1,A0 +merge 01#A1,A2,A3,A0 +merit 01#A1,A0 +mesh 01#A1,A2,A0 +mess 01#A1,A0 02#A1,A0 +metabolize 01#A1,A2,A0 +miff 01#A1,A2,A0 +migrate 01#A1,A0 +militate 01#A1,A0 +milk 01#A1,A2,A0 02#A1,A2,A0 +mill 01#A1,A2,A0 02#A1,A0 +mimic 01#A1,A0 +mince 01#A1,A0 02#A1,A2,A0 +mind 01#A1,A0 02#A1,A0 03#A0 +mine 01#A1,A0 +mingle 01#A1,A0 02#A1,A2,A0 +miniaturize 01#A1,A0 +minimize 01#A1,A0 +mint 01#A1,A2,A3,A4,A0 +mire 01#A1,A2 +mirror 01#A1,A2 +miscalculate 01#A1,A2,A0 +mishandle 01#A1,A0 +misinterpret 01#A1,A2,A0 +mislay 01#A1,A0 +mislead 01#A1,A2,A0 02#A1,A2,A0 +misplace 01#A1,A0 +misrepresent 01#A1,A2,A0 +miss 01#A1,A0 02#A1,A2,A0 03#A1,A0 +misspend 01#A1,A2,A0 +misstate 01#A1,A0 +mistake 01#A1,A2,A0 +mistreat 01#A1,A0 +misunderstand 01#A1,A0 +misuse 01#A1,A0 +mitigate 01#A1,A0 +mix 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +moan 01#A1,A0 +mobilize 01#A1,A0 +mock 01#A1,A2,A0 +model 01#A1,A2,A0 02#A1,A0 +moderate 01#A1,A0 +modernize 01#A1,A0 +modify 01#A1,A2,A3,A0 +modulate 01#A1,A2,A3,A0 +mold 01#A1,A2,A0 +mollify 01#A1,A2,A0 +monitor 01#A1,A2,A0 +monopolize 01#A1,A0 +moonlight 01#A1,A0 +mop 01#A1,A2,A0 +mortgage 01#A1,A2,A3,A0 +motivate 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +motorize 01#A1,A0 +mount 01#A1,A0 02#A1,A2,A3,A4,AM 03#A1,A0 +mourn 01#A1,A0 +mouth 01#A1,A2,A0 +move 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 04#A1,A0 +mow 01#A1,A0 +muck 01#A1 +muddle 01#A1,A0 +muddy 01#A1,A0 +muffle 01#A1,A2,A0 +mull 01#A1,A0 02#A1,A0 +multiply 01#A1,A2,A3,A4,A0 +mumble 01#A1,A2,A0 +munch 01#A1,A0 +murder 01#A1,A2,A0 +murmur 01#A1,A2,A0 +muscle 01#A1,A2,A0 +muse 01#A1,A0 +mushroom 01#A1,A2,A3,A4 +muster 01#A1,A0 +mutate 01#A1,A0 +mute 01#A1,A0 +mutilate 01#A1,A0 +mutter 01#A1,A2,A0 +muzzle 01#A1,A2,A0 +nag 01#A1,A2,A0 +nail 01#A1,A0 02#A1,A0 +name 01#A1,A2,A0 02#A1,A2,A0 +name-drop 01#A0 +namedrop 01#A0 +narrow 01#A1,A2,A3,A4,A0 +nationalize 01#A1,A0 +navigate 01#A1,A2,A0 +near 01#A1,A2 +necessitate 01#A1,A0 +need 01#A1,A0 +negate 01#A1,A0 +neglect 01#A1,A0 +negotiate 01#A1,A2,A0 +neighbor 01#A1 +net 01#A1,A2,A3 02#A1,A2,A0 03#A1,A0 +network 01#A1,A0 +neutralize 01#A1,A0 +nibble 01#A1,A0 +nick 01#A1,A3,A0 +nickname 01#A1,A2,A0 +nod 01#A1,A0 +nominate 01#A1,A2,A0 +normalize 01#A1,A0 +nose-dive 01#A1,A2,A3,A4,AM +nosedive 01#A1,A2,A3,A4,AM +notch 01#A1,A2,A0 02#A1,A2,A3,A4,AM,A0 +note 01#A1,A2,A3,A0 02#A1,A2 +notice 01#A1,A0 +notify 01#A1,A2,A0 +nudge 01#A1,A2,A3,A4,A5,AM 02#A1,A2,A0 +nullify 01#A1,A0 +number 01#A1,A2 +nurse 01#A1,A0 +nurture 01#A1,A0 +obey 01#A1,A0 +obfuscate 01#A1,A0 +object 01#A1,A0 +obligate 01#A1,A2,A0 +oblige 01#A0 02#A1,A2,A0 +obliterate 01#A1,A2,A0 +obscure 01#A1,A2,A0 +observe 01#A1,A0 02#A1,A2,A3,A0 +obsess 01#A1,A0 +obstruct 01#A1,A0 +obtain 01#A1,A2,A0 +obviate 01#A1,A0 +occasion 01#A1,A0 +occupy 01#A1,A2,A0 +occur 01#A1 +offend 01#A1,A2,A0 +offer 01#A1,A2,A3,A0 +offset 01#A1,A2,A0 +ogle 01#A1,A0 +omit 01#A1,A2,A0 +ooze 01#A1 +open 01#A1,A2,A3,A0 02#A1,AM 03#A1,A2,A3,A0 +operate 01#A1,A2,A3,A4,A0 +opine 01#A1,A2,A0 +oppose 01#A1,A0 +opt 01#A1,A0 +orchestrate 01#A1,A0 +ordain 01#A1,A0 +order 01#A1,A2,A0 02#A1,A2,A3,A0 +organize 01#A1,A2,A3,A4,A0 +orient 01#A1,A0 +originate 01#A1,A2,A4,A0 +orphan 01#A1,A0 +oust 01#A1,A2,A0 +outbid 01#A1,A2,A0 +outdistance 01#A1,A0 +outdo 01#A1,A0 +outfit 01#A1,A2,A0 +outflank 01#A1,A2 +outgrow 01#A1,A0 +outlast 01#A1,A0 +outlaw 01#A1,A0 +outline 01#A1,A0 +outnumber 01#A1,A2,A3 +outpace 01#A1,A2,A0 +outperform 01#A1,A2,A0 +outrage 01#A1,A0 +outrank 01#A1,A0 +outsell 01#A1,A2,A0 +outshine 01#A1,A0 +outstrip 01#A1,A2,A0 +outweigh 01#A1,A0 +overarch 01#A1 +overbid 01#A0 02#A1,A0 +overcome 01#A1,A0 +overdo 01#A1,A0 +overdose 01#A1,A0 +overemphasize 01#A1,A2,A0 +overflow 01#A1,A2 +overhang 01#A1,A0 +overhaul 01#A1,A2,A3,A0 +overheat 01#A1,A0 +overlap 01#A1,A2,A0 +overlay 01#A1,A2,A0 +overlook 01#A1,A0 02#A1,A0 +overpay 01#A1,A2,A0 +overpower 01#A1,A0 +overprice 01#A1,A0 +overreact 01#A1,A0 +override 01#A1,A0 +overrule 01#A1,A0 +oversee 01#A1,A0 +oversell 01#A1,A0 +overshadow 01#A1,A0 +oversimplify 01#A1,A0 +overstate 01#A1,A2,A0 +overstrain 01#A1,A0 +overtake 01#A1,A0 +overtax 01#A1,A2,A3,A0 +overthrow 01#A1,A0 +overturn 01#A1,A0 +overuse 01#A1,A0 +overvalue 01#A1,A0 +overwhelm 01#A1,A2,A0 +overwork 01#A1,A0 +owe 01#A1,A2,A3,A0 +own 01#A1,A0 02#A1,A0 +pace 01#A1,A0 02#A1,A0 +pacify 01#A1,A0 +pack 01#A1,A2,A0 02#A1,A0 +package 01#A1,A2,A3,A0 +paint 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +pair 01#A1,A2,A0 02#A1,A2,A0 +pale 01#A1 02#A1,A2 +palm 01#A1,A0 +pamper 01#A1,A0 +pan 01#A1,A0 02#A0 +panic 01#A1 +pant 01#A0 02#A1,A2,A0 +parachute 01#AA,A1,A0 +parallel 01#A1,A0 +paralyze 01#A1,A0 +parcel 01#A1,A2,A0 +parch 01#A1,A0 +pardon 01#A1,A0 +pare 01#A1,A2,A3,A0 02#A1,A2,A3,A4,A0 03#A1,A2,A3,A0 04#A1,A2,A3,A4,A0 +park 01#A1,A2,A0 +parry 01#A1,A2,A0 +part 01#A1,A0 +partake 01#A1,A0 +participate 01#A1,A0 +pass 11#A0 01#A1,A2,A0 12#A1,A0 02#A1,A0 13#A1,A2,A0 03#A1 14#A1,A2,A0 04#A1,A0 15#A1 05#A1,A2,A0 06#A1,A2 07#A1,A2 08#A1,A0 09#A1,A2,A0 10#A1,A2,A0 +paste 01#A1,A2,A3,A0 02#A1 +pasteurize 01#A1,A0 +pat 01#A1,A0 +patch 01#A1,A0 02#A1,A0 +patent 01#A1,A0 +patrol 01#A1,A0 +patronize 01#A1,A0 +pattern 01#A1,A2,A0 +pause 01#A0 +pave 01#A1,A2,A0 02#A1,A2,A0 +paw 01#A1,A0 +pawn 01#A1,A0 +pay 01#A1,A2,A3,A0 02#A1,A0 03#A1,A2,A3,A0 04#A1 05#A1 06#A1,A0 +peak 01#A1,A2,A3 +peal 01#A1,A0 +peck 01#A1,A0 02#A1,A2,A3,A0 +pedal 01#A1,A2,A0 +peddle 01#A1,A2,A3,A0 +peek 01#A1,A0 +peel 01#A1,A0 02#A1,A0 +peer 01#A1,A0 +peg 01#A1,A2,A0 +pen 01#A1,A2,A0 02#A1,A0 +penalize 01#A1,A2,A0 +penetrate 01#A1,A0 +people 01#A1,A2 +pepper 01#A1,A2,A0 +perceive 01#A1,A2,A0 +perch 01#A1,A2,A0 +perfect 01#A1,A0 +perform 01#A1,A2,A0 +perk 01#A1,A0 +permeate 01#A1,A0 +permit 01#A1,A2,A0 +perpetrate 01#A1,A2,A0 +perpetuate 01#A1,A0 +persecute 01#A1,A0 +persist 01#A1,A0 +personalize 01#A1,A0 +persuade 01#A1,A2,A0 +pertain 01#A1,A0 +perturb 01#A1,A0 +peruse 01#A1,A0 +pervade 01#A1,A0 +pester 01#A1,A0 +petition 01#A1,A0 +phase 01#A1,A0 +phone 01#A1,A2,A0 +photocopy 01#A1,A0 +photograph 01#A1,A0 +pick 01#A1,A2,A3,A0 02#A1,A0 03#A1,A2,A0 04#A1,A2,A3,A4,A0 05#A1,A2,A3,A4 06#A1,A0 07#A1,A0 08#A1,A2,A0 09#A1,A0 +picture 01#A1,A0 +piece 01#A1,A2,A3,A0 +piggyback 01#A1,A0 +pile 01#A1,A2,A0 02#A1,A0 03#A1,A2 04#A1,A2,A0 +pillory 01#A1,A0 +pin 01#A1,A2,A0 +pinch 01#A1,A2,A0 02#A1,A2,A0 +pine 01#A1,A0 02#A1,A0 +ping 01#A0 +pinpoint 01#A1,A0 +pioneer 01#A1,A0 +pipe 01#A1,A2,A3,A0 02#A1,A0 03#A0 +pirate 01#A1,A2,A0 +pit 01#A1,A2,A0 02#A1,A0 +pitch 01#A1,A2,A0 02#A1,A2,A0 03#A0 04#A1,A2,A3,A0 +pivot 01#A1,A2,A0 +placate 01#A1,A2,A0 +place 01#A1,A2,A0 +plague 01#A1,A2,A0 +plan 01#A1,A2,A3,A0 +plant 01#A1,A2,A0 +play 01#A1,A2,A0 02#A1,A0 03#A1,A0 04#A1,A0 05#A1,A2,A0 06#A1,A0 07#A1,A0 +plead 01#A1,A2,A0 02#A1,A2,A0 +please 01#A1,A2,A0 +pledge 01#A1,A2,A0 +plot 01#A1,A0 02#A1,A0 +plow 01#A1,A2,A0 +pluck 01#A1,A2,A0 +plug 01#A1,A2,A0 02#A1,A0 03#A1,A0 04#A1,A2,A0 +plummet 01#A1,A2,A3,A4,AM +plunge 01#A1,A2,A3,A4,AM 02#A1,A2,A0 +plunk 01#A1,A2,A0 +ply 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 +poach 01#A1,A2,A0 02#A1,A0 +pocket 01#A1,A0 +pockmark 01#A1,A0 +point 01#A1,A2,A0 02#A1,A0 +poise 01#A1,A2,A0 +poison 01#A1,A2,A0 02#A1,A2,A0 +poke 01#A1,A2,A0 02#A1,A2,A0 +polarize 01#A1,A0 +police 01#A1,A0 +polish 01#A1,A2,A3,A0 02#A1,A0 +politicize 01#A1,A0 +poll 01#A1,A2,A0 +pollinate 01#A1,A0 +pollute 01#A1,A0 +ponder 01#A1,A0 +pontificate 01#A1,A2,A0 +pool 01#A1,A2,A0 02#A1 +pop 01#A0 02#A1,A0 03#A1 04#A1,A2,A3,A4,A5,A0 +popularize 01#A1,A2,A0 +populate 01#A1,A2,A0 +pore 01#A1,A0 +port 01#A1,A0 +portend 01#A1,A2,A0 +portray 01#A1,A2,A0 +pose 01#A1,A0 02#A1,A2,A0 +position 01#A1,A2,A0 +possess 01#A1,A0 +post 01#A1,A2,A0 +postmark 01#A1,A2,A0 +postpone 01#A1,A2,A3,A4,A0 +posture 01#A1,A0 +pounce 01#A1,A0 +pound 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 +pour 01#A1,A2,A3,A0 +power 01#A1,A2,A0 +practice 01#A1,A2,A0 +praise 01#A1,A2,A0 +prance 01#A1,A0 +pray 01#A1,A2,A0 +pre-empt 01#A1,A0 +preach 01#A1,A0 +prearrange 01#A1,A0 +precede 01#A1,AA,A0 +precipitate 01#A1,A0 +preclude 01#A1,A2,A3,A0 +predate 01#A1,A2 +predetermine 01#A1,A0 +predicate 01#A1,A2,A0 +predict 01#A1,A0 +predispose 01#A1,A2,A0 +prefer 01#A1,A2,A0 +preform 01#A1,A2,A3,A4,A0 +prejudice 01#A1,A2,A0 +premiere 01#A1,A0 +preoccupy 01#A1,A2,A0 +prepare 01#A1,A2,A3,A0 02#A1,A2,A0 +prepay 01#A1,A2,A3,A0 +presage 01#A1,A0 +prescribe 01#A1,A0 02#A1,A2,A0 +present 01#A1,A2,A3,A0 +preserve 01#A1,A2,A0 +preside 01#A1,A0 +press 01#A1,A2,A0 02#A1,Am,A0 +pressure 01#A1,A2,A0 02#A1,Am,A0 +presume 01#A1,A2,A0 +pretend 01#A1,A0 +prevail 01#A1,A2,A0 02#A1 +prevent 01#A1,A2,A3,A0 +preview 01#A1,A0 +prey 01#A1,A0 +price 01#A1,A2,A0 +prick 01#A1,A0 +prime 01#A1,A2,A0 +print 01#A1,A2,A0 +privatize 01#A1,A0 +prize 01#A1,A2,A0 +probe 01#A1,A2,A0 +proceed 01#A1,A0 +process 01#A1,A2,A3,A0 +proclaim 01#A1,A2,A0 +prod 01#A1,A2,A0 +produce 01#A1,A2,A3,A0 +profess 01#A1,A2,A3,A0 +proffer 01#A1,A2,A0 +profile 01#A1,A0 +profit 01#A1,A0 +profiteer 01#A0 +program 01#A1,A2,A0 +progress 01#A1,A3,A4 +prohibit 01#A1,A2,A3,A0 +project 01#A1,A2,A0 02#A1,A0 +proliferate 01#A0 +prolong 01#A1,A2,A3,A4,A0 +promise 01#A1,A2,A0 +promote 01#A1,A2,A3,A0 02#A1,A2,A0 +prompt 01#A1,A0 02#A1,A2,A0 03#A1,A0 +promulgate 01#A1,A0 +pronounce 01#A1,A2,A0 +proofread 01#A1,A0 +prop 01#A1,A2,A0 +propagandize 01#A1,A0 +propel 01#A1,A2,A0 +propose 01#A1,A2,A3,A0 +proscribe 01#A1,A0 +prosecute 01#A1,A2,A0 +prosper 01#A0 +protect 01#A1,A2,A3,A0 +protest 01#A1,A0 02#A1,A2,A0 +prove 01#A1,A2,A0 +provide 01#A1,A2,A0 +provision 01#A1,A2,A0 +provoke 01#A1,A2,A0 +prune 01#A1,A2,A3,A0 +pry 01#A1,A2,A0 +publicize 01#A1,A2,A0 +publish 01#A1,A0 +puke 01#A0 02#A1,A0 +pull 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A0 04#A1,A0 05#A1,A2,A0 06#A1,A2,A0 +pulverize 01#A1,A0 +pummel 01#A1,A2,A0 +pump 01#A1,A2,A0 02#A1,A0 03#A1,A2,A3,A0 +punch 01#A1,A0 +punish 01#A1,A2,A0 +purchase 01#A1,A2,A3,A4,A0 +purge 01#A1,A2,A0 +purport 01#A1,A0 +purr 01#A0 02#A1,A2,A0 +pursue 01#A1,A0 +push 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2 +put 11#A1,A2,A0 01#A1,A2,A0 12#A1,A0 02#A1,A0 03#A1,A2,A0 04#A1,A0 05#A1,A0 06#A1,A2,A0 07#A1,A0 08#A1,A0 09#A1,A0 10#A1,A0 +puzzle 01#A1,A2,A0 +pyramid 01#A1,A0 +quack 01#A0 +quadruple 01#A1,A2,A3,A4,A0 +qualify 01#A1,A2,A0 02#A1,A2 +quantify 01#A1,A0 +quarrel 01#A1,A2,A0 +quash 01#A1,A0 +quell 01#A1,A0 +query 01#A1,A0 +question 01#A1,A2,A0 +queue 01#A1,A0 +quibble 01#A1,A2,A0 +quicken 01#A1,A2,A3,A4 +quiet 01#A1,A0 02#A1,A0 +quip 01#A1,A2,A0 +quit 01#A1,A0 +quiz 01#A1,A2,A0 +quote 01#A1,A2,A0 +race 01#A1,A0 02#A1,A2,A0 +rack 01#A1,A2,A3,A4,A0 +radio 01#A1,A2,A0 +rage 01#A1,A2,A0 02#A1,A0 +raid 01#A1,A0 +rail 01#A1,A0 +rain 01#A1,A2,A0 +raise 01#A1,A2,A3,A4,AM,A0 02#A1,A2,A0 +rake 01#A1,A2,A3,A0 02#A1,A0 +rally 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +ram 01#A1,A2,A0 +ramble 01#A1,A0 02#A1,A0 +ramp 01#A1,A0 +range 01#A1,A2,A3,A4,A5 +rank 01#A1,A2,A0 +rankle 01#A1,A0 +rape 01#A1,A0 +rarefy 01#A1,A0 +rat 01#A1,A0 +ratchet 01#A1,A2,A3,A4,A0 +rate 01#A1,A2,A0 +ratify 01#A1,A2,A0 +ration 01#A1,A2,A0 02#A1,A2,A0 +rationalize 01#A1,A2,A0 +rattle 01#A1,A0 02#A0 +ravage 01#A1,A2,A0 +rave 01#A1,A0 +raze 01#A1,A2,A0 +re-create 01#A1,A2,A3,A4,A0 +re-elect 01#A1,A2,A0 +re-emerge 01#A1,A0 02#A1,A0 +re-emphasize 01#A1,A2,A0 +re-enter 01#A1,A0 02#A1,A2,A3,A0 +re-evaluate 01#A1,A0 +reach 01#A1,A0 02#A1,A0 03#A1,A2,A0 +react 01#A1,A2,A0 +reactivate 01#A1,A0 +read 01#A1,A2,A3,A0 +readmit 01#A1,A2,A0 +reaffirm 01#A1,A0 +realign 01#A1,A2,A0 +realize 01#A1,A0 02#A1,A2,A0 +reallocate 01#A1,A2,A3,A0 +reap 01#A1,A2,A0 +reapportion 01#A1,A2,A3,A0 +reappraise 01#A1,A2,A0 +rear 01#A1,A0 +rearm 01#A1,A0 +rearrange 01#A1,A0 +reason 01#A1,A0 +reassert 01#A1,A0 02#A1,A2,A3,A0 03#A1,A2,A3,A0 +reassess 01#A1,A0 +reassign 01#A1,A2,A0 +reassure 01#A1,A2,A0 +reawaken 01#A1,A2,A0 +rebel 01#A1,A0 +rebound 01#A1,A2,A3,A4 +rebuff 01#A1,A0 +rebuild 01#A1,A0 +rebuke 01#A1,A2,A0 +rebut 01#A1,A0 +recalculate 01#A1,A2,A0 +recall 01#A1,A2,A0 02#A1,A2,A3,A0 +recant 01#A1,A0 +recapture 01#A1,A0 +recede 01#A0 +receive 01#A1,A2,A3,A4,A0 +recess 01#A1,A0 02#A0 +recharge 01#A1,A2,A0 +recite 01#A1,A0 +reckon 01#A1,A2,A0 02#A1,A0 +reclaim 01#A1,A2,A0 +reclassify 01#A1,A2,A0 02#A1,A0 +recognize 01#A1,A2,A0 02#A1,A0 +recommend 01#A1,A2,A3,A0 +reconcile 01#A1,A2,A0 +reconsider 01#A1,A0 +reconstruct 01#A1,A2,A0 +record 01#A1,A0 +recount 01#A1,A0 +recoup 01#A1,A2,A0 +recover 01#A1,A2 02#A1,A2,A0 03#A1,A2,A0 +recreate 01#A1,A2,A3,A4,A0 +recruit 01#A1,A2,A3,A0 +rectify 01#A1,A0 +recur 01#A1,A0 +recycle 01#A1,A2,A0 +redden 01#A1,A0 +redeem 01#A1,A2,A0 +redefine 01#A1,A2,A0 +redeploy 01#A1,A2,A0 +redesign 01#A1,A2,A0 +redevelop 01#A1,A2,A3,A4,A0 +redirect 01#A1,A2,A0 +rediscover 01#A1,A0 +redistribute 01#A1,A0 +redline 01#A1,A2,A0 +redo 01#A1,A2,A3,A4,A0 +redouble 01#A1,A2,A3,A4,A0 +redound 01#A1,A0 +redress 01#A1,A0 +reduce 01#A1,A2,A3,A4,AM,A0 +reel 01#A1,A2 02#A1,A0 +reestablish 01#A1,A0 +reevaluate 01#A1,A0 +reexamine 01#A1,A0 +refashion 01#A1,A0 +refer 01#A1,A2,A0 02#A1,A2,A0 +refile 01#A1,A2,A3,A4,A0 02#A1,A0 +refinance 01#A1,A2,A0 02#A1,A2,A0 +refine 01#A1,A2,A0 +refit 01#A1,A2,A3,A0 +reflect 01#A1,A2 02#A1,A0 +refocus 01#A1,A2,A0 +reform 01#A1,A2,A3,A0 +reformulate 01#A1,A0 +refrain 01#A1,A0 +refresh 01#A1,A2,A0 +refuel 01#A1,A0 +refund 01#A1,A2,A3,A0 +refurbish 01#A1,A0 +refuse 01#A1,A2,A0 +refute 01#A1,A0 +regain 01#A1,A2,A0 +regard 01#A1,A2,A0 +regenerate 01#A1,A0 +register 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +regret 01#A1,A0 +regroup 01#A0 +regulate 01#A1,A0 +rehabilitate 01#A1,A0 +rehash 01#A1,A0 +rehearse 01#A1,A0 +reign 01#A1,A0 +reignite 01#A1,A0 +reimburse 01#A1,A2,A3,A0 +reimpose 01#A1,A2,A0 +rein 01#A1,A0 +reinforce 01#A1,A2,A0 +reinstall 01#A1,A0 +reinstate 01#A1,A2,A0 +reintegrate 01#A1,A2,A0 +reintroduce 01#A1,A2,A0 02#A1,A2,A0 +reinvigorate 01#A1,A0 +reiterate 01#A1,A2,A0 +reject 01#A1,A3,A0 +rejoin 01#A1,A2,A0 +rejuvenate 01#A1,A0 +rekindle 01#A1,A0 +relate 01#A1,A2,A0 02#A1,A2,A3,A0 +relax 01#A1,A0 +relay 01#A1,A2,A0 +release 01#A1,A2,A0 +relegate 01#A1,A2,A0 +relent 01#A1,A0 +relieve 01#A1,A2,A0 02#A1,A2,A0 +relinquish 01#A1,A0 +relish 01#A1,A0 +relive 01#A1,A0 +relocate 01#A1,A2,A3,A0 +rely 01#A1,A2,A0 +remain 01#A1,A2,A3 +remake 01#A1,A2,A3,A0 +remand 01#A1,A2,A0 +remark 01#A1,A2,A0 +remedy 01#A1,A2,A0 +remember 01#A1,A2,A0 +remind 01#A1,A2,A0 +remodel 01#A1,A0 +remove 01#A1,A2,A0 +remunerate 01#A1,A2,A0 +rename 01#A1,A2,A0 +render 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +rendezvous 01#A1,A0 +renege 01#A1,A0 +renegotiate 01#A1,A2,A0 +renew 01#A1,A0 +renounce 01#A1,A2,A0 +renovate 01#A1,A0 +rent 01#A1,A2,A3,A4,A0 02#A1,A2,A3,A4,A0 +reopen 01#A1,A2,A3,A0 +reorganize 01#A1,A2,A3,A4,A0 +reorient 01#A1,A0 +repaint 01#A1,A2,A0 02#A1,A2,A0 +repair 01#A1,A0 +repatriate 01#A1,A2,A0 +repay 01#A1,A2,A3,A0 +repeal 01#A1,A0 +repeat 01#A1,A2,A3,A0 +repel 01#A1,A2,A0 +replace 01#A1,A2,A0 +replenish 01#A1,A2,A0 +replicate 01#A1,A0 +reply 01#A1,A2,A0 +report 01#A1,A2,A0 +reposition 01#A1,A2,A0 +repossess 01#A1,A0 +represent 01#A1,A0 +repress 01#A1,A0 +reprint 01#A1,A0 +reproduce 01#A1,A0 +reprove 01#A1,A2,A0 +repudiate 01#A1,A0 +repute 01#A1,A2,A0 +request 01#A1,A2,A0 +require 01#A1,A2,A0 +requisition 01#A1,A0 +reschedule 01#A1,A2,A3,A0 +rescind 01#A1,A0 +rescue 01#A1,A2,A0 +research 01#A1,A0 +resell 01#A1,A2,A3,A4,A0 +resemble 01#A1,A2 +resent 01#A1,A2,A0 +reserve 01#A1,A2,A3,A0 +reset 01#A1,A2,A0 +reshape 01#A1,A0 +reshuffle 01#A1,A0 +reside 01#A1,A0 +resign 01#A1,A0 02#A1,A2 +resist 01#A1,A0 +resolve 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +resonate 01#A1,A2 +resort 01#A1,A0 +respect 01#A1,A2,A0 +respond 01#A1,A2,A0 +rest 01#A1,A0 02#A1,A2,A0 +restart 01#A1,A0 +restate 01#A1,A2,A3,A0 02#A1,A2,A3,A4,A0 +restore 01#A1,A2,A0 02#A1,A2,A0 +restrain 01#A1,A2,A3,A0 +restrict 01#A1,A2,A0 +restructure 01#A1,A2,A3,A0 +resubmit 01#A1,A2,A3,A0 +result 01#A1,A2 +resume 01#A1,A2,A0 +resurface 01#A1,A0 02#A1 +resurge 01#A0 +resurrect 01#A1,A0 +resuscitate 01#A1,A0 +retail 01#A1,A2,A3,A4,A0 +retain 01#A1,A3,A0 +retake 01#A1,A0 +retaliate 01#A1,A0 +retard 01#A1,A0 +retest 01#A1,A2,A0 +rethink 01#A1,A0 +retire 01#A1,A0 02#A1,A0 +retool 01#A1,A2,A3,A0 +retrace 01#A1,A0 +retract 01#A1,A0 +retrain 01#A1,A2,A0 +retreat 01#AA,A1,A0 02#A1,A2,A3,A4 +retrench 01#A0 +retrieve 01#A1,A2,A3,A0 +retrofit 01#A1,A2,A0 +retry 01#A1,A0 +return 01#A1,A2,A3,A4,AM 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2,A0 +reunite 01#A1,A2,A3,A0 +rev 01#A1,A0 02#A1,A2,A0 +revalue 01#A1,A2,A0 +revamp 01#A1,A0 +reveal 01#A1,A2,A3,A0 +revel 01#A1,A0 +reverberate 01#A1 +reverse 01#A1,A0 +revert 01#A1,A2 +review 01#A1,A2,A0 +revise 01#A1,A2,A3,A0 +revisit 01#A1,A0 +revitalize 01#A1,A0 +revive 01#A1,A0 +revoke 01#A1,A0 +revolutionize 01#A1,A0 +revolve 01#A1,A2 +reward 01#A1,A2,A0 +rework 01#A1,A0 +rewrite 01#A1,A0 +rhyme 01#A1,A2,A0 +rid 01#A1,A2,A0 +ride 01#A1,A0 02#A1,A0 +ridicule 01#A1,A2,A0 +riff 01#A1,A0 +rig 01#A1,A0 02#A1,A0 +right 01#A1,A0 +rile 01#A1,A0 +rim 01#A1,A2,A0 +ring 01#A1,A2,A0 02#A1,A2 03#A1,A0 +rip 01#A1,A0 02#A1 03#A1,A0 04#A1,A0 +ripen 01#A1,A0 +ripple 01#A1,A0 +rise 01#A1,A2,A3,A4,AM +risk 01#A1,A0 +rival 01#A1,A2,A0 +rivet 01#A1,A2,A0 +roam 01#A1,A0 +roar 01#A1,A2,A0 02#A1,A0 +rob 01#A1,A0 +rock 01#A1,A0 +rocket 01#A1,A2,A3,A4,AM +roil 01#A1,A0 +roll 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 +romance 01#A1,A0 +romanticize 01#A1,A0 +romp 01#A1,A0 +roost 01#A0 +root 01#A1,A0 02#A1,A2,A0 03#A1,A0 +rot 01#A1,A0 +rotate 01#A1,A0 02#A1,A2,A3,A0 +rough 01#A1,A0 +round 01#A1,A0 02#A1,A0 +route 01#A1,A2,A0 +row 01#A1,A0 +rub 01#A1,A2,A3,A0 +ruffle 01#A1,A0 02#A1,A0 +ruin 01#A1,A2,A0 +rule 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A0 +rumble 01#A0 +ruminate 01#A1,A0 +rumor 01#A1,A2,A0 02#A1,A0 +run 01#A1,A2,A3,A4,A0 02#A1,A0 03#A1,A2,A3 04#A1,A2,A3,A4,Am 05#A1,A2 06#A1,A0 07#A1,A0 08#A1,A2,A3,A0 09#A1,A2,A0 +rupture 01#A1,A0 +rush 01#A1,A2,A0 +rust 01#A1 +rusticate 01#A1,A0 +sabotage 01#A1,A2,A0 +sack 01#A1,A0 +sacrifice 01#A1,A2,A0 +saddle 01#A1,A2,A0 +safeguard 01#A1,A2,A0 +sag 01#A1,A2,A3,A4 +sail 01#A1,A0 +salt 01#A1,A2,A0 02#A1,A0 +salute 01#A1,A2,A0 +salvage 01#A1,A2,A0 +salve 01#A1,A0 +sample 01#A1,A0 +sanction 01#A1,A0 +sandwich 01#A1,A2,A0 +sanitize 01#A1,A0 +sap 01#A1,A2,A0 +satisfy 01#A1,A2,A0 02#A1,A2,A0 +saturate 01#A1,A2,A0 +save 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2,A0 +savor 01#A1,A0 +say 01#A1,A2,A3,A0 +scale 01#A1,A2,A3,A4,A5,A0 02#A1,A2,A0 03#A1,A0 +scamper 01#A0 +scan 01#A1,A2,A0 +scandalize 01#A1,A2,A0 +scare 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +scarf 01#A1,A0 +scatter 01#A1,A2,A0 +schedule 01#A1,A2,A3,A0 +scoff 01#A1,A0 +scold 01#A1,A2,A0 +scoop 01#A1,A2,A0 02#A1,A0 +scoot 01#AA,A1,A0 +score 01#A1,A2,A0 02#A1,A2,A0 +scorn 01#A1,A0 +scotch 01#A1,A0 +scour 01#A1,A2,A0 02#A1,A2,A0 +scout 01#A1,A0 +scowl 01#A1,A2,A0 +scramble 01#A1,A0 02#A1,A3,A0 03#Am,A0 +scrap 01#A1,A0 02#A1,A0 +scrape 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +scratch 01#A1,A2,A0 02#A1,A2,A3,A0 03#A1,A2,A0 04#A1,A0 +scream 01#A1,A2,A0 +screech 01#A1,A0 02#A1,A2 +screen 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 +screw 01#A1,A0 +scribble 01#A1,A2,A0 +scrimp 01#A1,A0 +scrounge 01#A1,A0 +scrutinize 01#A1,A2,A0 +scurry 01#A0 +scuttle 01#A1,A2,A0 02#A0 +seal 01#A1,A0 +search 01#A1,A0 +seat 01#A1,A2,A0 02#A1,A0 +secede 01#A1,A0 +second-guess 01#A1,A0 +secure 01#A1,A2,A3,A0 02#A1,A2,A0 +seduce 01#A1,A2,A0 +see 01#A1,A0 02#A1,A2,A0 03#A0 04#A1,A2,A0 +seed 01#A1,A0 02#A1,A2,A0 +seek 01#A1,A2,A0 02#A1,A2,A0 +seem 01#A1,A2 +seep 01#A1 +seethe 01#A1,A2,A0 +segment 01#A1,A2,A0 +segregate 01#A1,A2,A0 +seize 01#A1,A2,A0 +select 01#A1,A2,A3,A0 +self-destruct 01#A0 +sell 01#A1,A2,A3,A4,A0 02#A0 03#A1,A2,A3,A4,A0 +send 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A3,A4,A5,A0 04#A1,A0 +sense 01#A1,A2,A0 +sensitize 01#A1,A2,A0 +sentence 01#A1,A2,A3,A0 +separate 01#A1,A2,A0 +sequester 01#A1,A0 +serve 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +service 01#A1,A0 +set 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A0 05#A1,A0 06#A1,A0 07#A1,A0 08#A1,A2 +settle 01#A1,A2,A3,A0 02#A1,A2,A3,A0 03#A1,A2,A3,A4,A0 04#A1 +sever 01#A1,A0 +sew 01#A1,A2,A3,A0 02#A1,A2,A4,A0 03#A1,A0 +shadow 01#A1,A0 +shag 01#A1,A0 +shake 01#A1,A0 02#A1,A2,A0 03#A1,A0 +shape 01#A1,A2,A0 02#A1,A2 03#A1 +share 01#A1,A2,A0 02#A1,A2,A0 +sharpen 01#A1,A2,A0 +shatter 01#A1,A2,A3,A0 +shave 01#A1,A2,A3,A0 +shear 01#A1,A2,A3,A0 +shed 01#A1,A0 02#A1,A2,A3,A4,AM +shell 01#A1,A0 02#A1,A2,A0 03#A1,A2,A3,A0 +shelter 01#A1,A2,A3,A0 +shelve 01#A1,A0 +shepherd 01#A1,A0 +shield 01#A1,A2,A0 +shift 01#A1,A2,A3,A0 +shimmer 01#A1 02#A1,A0 +shine 01#A1,A2,A0 02#A1 +ship 01#A1,A2,A3,A0 02#A1,A2 +shirk 01#A1,A0 +shiver 01#A1 +shock 01#A1,A2,A0 +shoehorn 01#A1,A2,A0 +shoot 01#A1,A2,A0 02#A1,A2,A3,A0 03#A1,A0 04#A1,A0 05#A1,A2,A0 06#A1,A2,A3,A4,A5,A0 07#A1,A2,A0 +shop 01#A1,A4,A0 02#A1,A2,A3,A4,A0 +shore 01#A1,A0 +short 01#A1,A0 02#A1,A0 03#A1,A2,A0 +short-circuit 01#A1,A0 +shorten 01#A1,A2,A3,A4,A0 +shoulder 01#A1,A0 +shout 01#A1,A2,A0 +shove 01#A1,A2,A0 +show 01#A1,A2,A0 02#A1 +shower 01#A1,A2,A0 +shred 01#A1,A2,A3,A0 +shrink 01#A1,A2,A3,A4,A0 +shrivel 01#A1,A2,A3,A4,A0 +shroud 01#A1,A2,A0 +shrug 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 +shudder 01#A1,A2 +shuffle 01#A0 02#AA,A1,A0 03#A1,A0 04#A1,A0 05#A1,A0 +shun 01#A1,A0 +shut 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A0 05#A1,A2,AM,A0 +shutter 01#A1,A0 +shuttle 01#A1,A2,A3,A0 02#A1,A2,A0 +shy 01#A1,A0 +side 01#A1,A0 02#A1,A2,A0 +sidestep 01#A1,A0 +sidetrack 01#A1,A0 +sift 01#A1,A0 +sigh 01#A1,A2,A0 02#A0 +sign 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A3,A0 04#A1,A0 +signal 01#A1,A2,A0 +signify 01#A1,A0 +silence 01#A1,A0 +silt 01#A1,A0 +simmer 01#A1,A0 +simplify 01#A1,A0 +simulate 01#A1,A2,A0 +sing 01#A1,A2,A0 +single 01#A1,A2,A3,A0 +sink 01#A1,A2,A3,A4,A5,A0 +sip 01#A1,A2,A0 +siphon 01#A1,A2,A0 02#A1,A2,A0 +sit 01#A1,A2 02#A1 03#A1 +situate 01#A1,A2,A0 +size 01#A1,A2,A0 02#A1,A0 +sizzle 01#A0 +sketch 01#A1,A2,A0 02#A1,A0 +skew 01#A1,A0 +ski 01#A1,A0 +skid 01#A1,A0 02#A1,A2,A3,A4 +skim 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 +skimp 01#A1,A0 +skip 01#A1,A0 02#A0 03#A0 +skipper 01#A1,A0 +skirmish 01#A1,A0 +skirt 01#A1,A2 02#A1,A0 +skyrocket 01#A1,A2,A3,A4,AM +slack 01#A1,A2 +slacken 01#A1,A0 +slam 01#A1,A2,A0 02#A1,A2,A0 +slap 01#A1,A2,A0 +slash 01#A1,A2,A3,A0 02#A1,A2,A3,A4,A0 +slate 01#A1,A2,A0 +slaughter 01#A1,A0 +slay 01#A1,A0 +sleep 01#A1,A2,A0 02#A1,A0 +slice 01#A1,A2,A0 02#A1,A0 +slide 01#A1,A0 02#A1,A2,A3,A4,AM +slim 01#A1,A0 +slip 01#A1,A2,A3,A4,AM 02#A1,A2,A0 03#A1,A2,A0 04#A1,A0 +slither 01#A0 +sliver 01#A2,A0 +slog 01#A1,A0 +slope 01#A0 +slosh 01#A1,A2,A0 +slow 01#A1,A0 02#A1,A3,A4,AM 03#A1,A0 +slump 01#A1,A2,A3,A4,A0 +smack 01#A1,A0 +smart 01#A1,A2 +smash 01#A1,A0 02#A1 +smell 01#A1,A0 02#A1,A2 +smile 01#A1,A2,A0 +smoke 01#A1,A0 02#A1,A0 03#A1,A0 +smolder 01#A0 +smooth 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 +smother 01#A1,A2,A0 +smuggle 01#A1,A2,A3,A0 +snag 01#A1,A2,A0 +snake 01#A1,A0 +snap 01#A1,A2,A0 03#A1,A2,A0 04#A0 05#A1,A2,A3,A0 06#A1,A2,A0 07#A1,A2,A0 08#A1,A2,A3,A4,A0 +snare 01#A1,A2,A0 +snatch 01#A1,A2,A0 +sneak 01#A1,A2,A3,A0 02#A1,A2,A0 +sniff 01#A1,A0 02#A1,A2,A0 03#A1,A0 +snipe 01#A1,A0 +snivel 01#A0 +snoop 01#A1,A0 +snore 01#A0 +snort 01#A1,A0 02#A1,A0 +snowball 01#A1,A2 +snub 01#A1,A0 +soak 01#A1,A0 02#A1,A2,A0 +soar 01#A1,A2,A3,A4,AM 02#A1 +sob 01#A0 +sober 01#A1,A2,A0 02#A1,A0 +socialize 01#A0 02#A1,A0 +sock 01#A1,A2,A0 02#A1,A2,A0 +soften 01#A1,A0 +soil 01#A1,A0 +solicit 01#A1,A2,A0 +solidify 01#A1,A0 +solve 01#A1,A2,A0 +soothe 01#A1,A0 +sort 01#A1,A2,A0 02#A1,A2,A0 +sound 01#A1,A2,A3 02#A1,A0 +sour 01#A1,A2,A0 +sow 01#A1,A0 +span 01#A1,A0 +spar 01#A1,A0 +spare 01#A1,A2,A0 02#A1,A0 +spark 01#A1,A2,A0 +spawn 01#A1,A0 +speak 01#A1,A2,A0 02#A0 03#A1,A0 +spearhead 01#A1,A0 +specialize 01#A1,A0 +specify 01#A1,A0 +speculate 01#A1,A0 +speed 01#A1,A0 02#A1,A0 +spell 01#A1,A2,A0 02#A1,A0 03#A1,A0 +spend 01#A1,A2,A3,A4,A0 02#A1,A2,A0 +spew 01#A1,A2,A0 +spike 01#A1,A0 +spill 01#A1,A2,A0 02#A1,A2 03#A1,A2 +spin 01#A1,A0 02#A1,A2,A3,A4,A0 03#A1,A0 04#A1,A2,A0 +spiral 01#A1,A2,A0 +spirit 01#A1 +spit 01#A1,A2,A0 +splash 01#A1,A2,A0 +split 01#A1,A2,A0 02#A1,A0 +spoil 01#A1,A0 +sponsor 01#A1,A0 +spook 01#A1,A2,A0 +sport 01#A1,A0 02#A1,A0 +spot 01#A1,A2,A3,A0 02#A1,A2,A0 +spotlight 01#A1,A0 +spout 01#A1,A0 +sprawl 01#A1,A2,A0 +spray 01#A1,A2,A0 +spread 01#A1,A2,A3,A0 02#A1,Am 03#A1,A0 +spring 01#A1,A2 02#A1 +sprinkle 01#A1,A2,A0 +sprout 01#A1,A2,A0 +spruce 01#A1,A0 +spur 01#A1,A2,A0 +spurn 01#A1,A0 +spurt 01#A1,A2 02#A1,A2,A3,A4 +sputter 01#A0 +spy 01#A1,A0 +squander 01#A1,A0 +square 01#A1,A0 02#A1,A2,A0 03#A1,A2,A3,A0 +squat 01#A1,A2 02#A1,A0 +squeak 01#A1,A0 +squeegee 01#A1,A0 +squeeze 01#A1,A0 02#A1,A2,A0 03#A1,A2,A3,A4,A0 +squelch 01#A1,A0 +squint 01#A1,A0 +squirm 01#A1,A0 +stab 01#A1,A2,A3,A0 +stabilize 01#A1,A2,A0 +stack 01#A1,A2,A0 02#A1 03#A1,A0 +staff 01#A1,A2,A0 +stage 01#A1,A0 +stagger 01#A1 02#A1,A0 +stagnate 01#A1 +stain 01#A1,A0 +stake 01#A1,A2,A0 02#A1,A2,A3,A0 +stalk 01#A0 02#A1,A0 +stall 01#A1,A0 02#A1,A0 +stamp 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 04#A1,A0 +stampede 01#A1,A2,A0 +stanch 01#A1,A0 +stand 01#A1,A2,A0 02#A1,A2 03#A1,A0 04#A1,A2 +standardize 01#A1,A0 +staple 01#A1,A2,A0 +star 01#A1,A2,A3 +stare 01#A1,A0 02#A1,A0 +start 01#A1,A2,A0 02#A1 04#A1,A2,A3,A0 +startle 01#A1,A0 +starve 01#A1,A2,A0 +stash 01#A1,A2,A0 02#A1,A0 +state 01#A1,A2,A3,A0 +station 01#A1,A2,A0 +stave 01#A1,A0 +stay 01#A1,A2,A3 02#A1 03#A1,A0 +steady 01#A1,A0 +steal 01#A1,A2,A0 +steam 01#A1,A0 02#A1 03#A1,A0 +steer 01#A1,A2,A0 02#A1,A0 +stem 01#A1,A2 02#A1,A0 +step 01#A1,A2,A3,A0 02#A1 03#A1,A0 04#A1,A0 +sterilize 01#A1,A0 +stew 01#A1,A0 02#A1,A0 +stick 01#A1,A2,A0 02#A1,A0 03#A0 04#A1,A0 +stifle 01#A1,A0 +stimulate 01#A1,A2,A0 +sting 01#A1,A2,A0 +stink 01#A1,A0 +stipulate 01#A1,A0 +stir 01#A1,A2,A0 02#A1,A2,A3,A0 03#A1,A2,A0 04#A1,A0 +stitch 01#A1,A2,A3,A0 02#A1,A2,A4,A0 +stock 01#A1,A2,A0 02#A1,A0 +stockpile 01#A1,A2,A0 +stoke 01#A1,A0 +stomach 01#A1,A0 +stomp 01#A1,A0 +stone 01#A1,A0 02#A1,A0 03#A1 +stonewall 01#A0 +stop 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 +store 01#A1,A2,A0 +storm 02#A1,A0 +stow 01#A1,A2,A0 +straddle 01#A1,A0 +strafe 01#A1,A0 +straighten 01#A1,A2,A0 02#A1,A0 +strain 01#A1,A0 02#A1,A2,A0 03#A1,A0 04#A1,A2 +strand 01#A1,A0 +strangle 01#A1,A0 +strap 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 +stray 01#A1,A0 +streak 01#A0 +stream 01#A1,A0 02#A1,Am +streamline 01#A1,A0 +strengthen 01#A1,A2,A3,A4,A0 +stress 01#A1,A2,A0 +stretch 01#A1,A2,A3,A4,A0 02#A1,A0 +strew 01#A1,A2,A0 +stride 01#A0 +strike 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 04#A1,A2,A3 05#A1,A2,A0 06#AA,A1,A0 07#A1,A2,A0 08#A1,A2,A0 09#A1,A2,A0 10#A1,A0 +string 01#A1,A0 +strip 01#A1,A2,A0 02#A1,A0 +strive 01#A1,A0 +stroke 01#A1,A2,A0 +stroll 01#A1,A0 +structure 01#A1,A2,A0 +struggle 01#A1,A2,A0 02#A1,A0 +stub 01#A1,A2,A0 +stud 01#A1,A2,A0 +study 01#A1,A0 +stuff 01#A1,A2,A0 +stumble 01#A1,A0 +stun 01#A1,A2,A0 +stunt 01#A1,A0 +style 01#A1,A2,A0 +stymie 01#A1,A0 +subcontract 01#A1,A2,A0 +subdue 01#A1,A0 +subject 01#A1,A2,A0 +sublet 01#A1,A2,A0 +submit 01#A1,A2,A3,A0 +subordinate 01#A1,A2,A0 +subpoena 01#A1,A0 +subscribe 01#A1,A0 02#A1,A0 +subside 01#A1,A2 +subsidize 01#A1,A0 +substantiate 01#A1,A0 +substitute 01#A1,A3,A4,A0 +subsume 01#A1,A2 +subtitle 01#A1,A2,A0 02#A1,A2,A0 +subtract 01#A1,A2,A0 +subvert 01#A1,A2,A0 +succeed 01#A1,A0 02#A1,A2,A0 03#A1,A2 +succumb 01#A1,A0 +suck 01#A1,A0 +sue 01#A1,A2,A3,A0 +suffer 01#A1,A0 +suffice 01#A0 +suggest 01#A1,A2,A0 +suit 01#A1,A2 02#A1,A0 +sum 01#A1,A2,A0 +summarize 01#A1,A2,A0 +summon 01#A1,A2,A0 02#A1,A0 +superimpose 01#A1,A2,A0 +supersede 01#A1,A0 +supervise 01#A1,A0 +supplement 01#A1,A2,A0 +supply 01#A1,A2,A0 +support 01#A1,A0 +suppose 01#A1,A2,A0 02#A1,A2 +suppress 01#A1,A0 +surface 01#A1 02#A1,A2,A0 +surge 01#A1,A2,A3,A4,AM +surmount 01#A1,A2,A0 02#A1,A0 +surpass 01#A1,A0 +surprise 01#A1,A2,A0 +surrender 01#A1,A2,A0 +surround 01#A1,A2,A0 +survey 01#A1,A2,A3,A0 +survive 01#A1,A0 02#A1,A2,A0 +suspect 01#A1,A2,A0 +suspend 01#A1,A2,A3,A0 02#A1,A2,A0 +sustain 01#A1,A0 02#A1,A0 +swallow 01#A1,A0 +swamp 01#A1,A2,A0 +swap 01#A1,A2,A3,A4,A0 +swathe 01#A1,A2,A0 +sway 01#A1,A2,A0 02#A1,A0 +swear 01#A1,A0 02#A0 03#A1,A0 04#A1,A2,A0 05#A1,A0 +sweat 01#A1,A0 02#A1,A2 +sweep 01#A1,A2,A3,A0 02#A1,A2,A3 +sweeten 01#A1,A2,A0 02#A1,A2,A3,A0 +swell 01#A1,A2,A3,A4,A0 +swerve 01#A1,A0 +swim 01#A1,A0 02#A1,A2 +swindle 01#A1,A2,A0 +swing 01#A1 02#A1,A0 03#A1,A2,A3,A4 04#A1 +switch 01#A1,A2,A3,A0 02#A1,A2,A3,A0 +swivel 01#A1,A0 +swoop 01#A0 02#A1,A0 +symbolize 01#A1,A2 +sympathize 01#A1,A0 +synchronize 01#A1,A2,A0 +syndicate 01#A1,A2,A0 +synthesize 01#A1,A2,A3,A4,A0 +taboo 01#A1,A0 +tack 01#A1,A2,A0 02#A1,A0 03#A1,A2,A3,A4 04#A1,A0 +tackle 01#A1,A2,A0 +tag 01#A1,A2,A0 +tail 01#A1,A0 +tailor 01#A1,A2,A0 +taint 01#A1,A2,A0 +take 11#A1,A0 01#A1,A2,A0 12#A1,A0 02#A1,A0 13#A1,A2,A0 03#A1,A2,A0 14#A1,A2 04#A1,A2,A0 15#A1,A2,A0 16#A1,A2,A0 05#A1,A0 17#A1,A0 06#A1,A0 07#A1,A0 18#A1,A2,A0 08#A1 19#A1,A2 09#A1,A2,A3,A4,A0 20#A1 10#A1,A2,A0 21#A1,A0 +talk 01#A1,A2,A0 02#A1,A2,A0 +tally 01#A1,A0 +tame 01#A1,A2,A0 +tamper 01#A1,A0 +tan 01#A1,A2,A0 +tangle 01#A1,A2,A0 02#A1,A2,A0 +tango 01#A1,A0 +tank 01#A0 +tap 01#A1,A2,A0 02#A1,A2,A3,A0 03#A1,A0 +tape 01#A1,A2,A0 02#A1,A0 +taper 01#A1 +tar 01#A1,A2,A0 +target 01#A1,A2,A0 +tarnish 01#A1,A0 +taste 01#A1,A0 02#A1,A2 +taunt 01#A1,A0 +tax 01#A1,A2,A3,A0 +teach 01#A1,A2,A0 +team 01#A1,A2,A0 02#A1,A2,A0 +tear 01#A1,A2,A0 02#A1,A0 03#A1 04#A1,A0 05#A1,A0 +tease 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 04#A1,A2,A0 +teem 01#A1,A0 +teeter 01#A1 +telegraph 01#A1,A2,A0 +telephone 01#A1,A2,A0 +televise 01#A1,A0 +tell 01#A1,A2,A0 02#A1,A2,A0 +temper 01#A1,A0 +tempt 01#A1,A2,A3,A0 +tend 01#A1,A0 02#A1,A2 +tender 01#A1,A2,A0 +term 01#A1,A3,A0 +terminate 01#A1,AM,A0 02#A1,A2,A0 +terrify 01#A1,A2,A0 +test 01#A1,A2,A0 +testify 01#A1,A2,A0 +tether 01#A1,A2,A0 +thank 01#A1,A2,A0 +theorize 01#A1,A0 +thin 01#A1,A0 02#A1,A0 +think 01#A1,A2,A0 +thrash 01#A1,A2,A0 02#A1,A2,A0 03#A1 +threaten 01#A1,A2,A0 +thrill 01#A1,A0 02#A1,A0 +thrive 01#A1,A2 +throw 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 +thrust 01#A1,A2,A0 +thumb 01#A1,A0 02#A1,A2,A0 +thunder 01#A1,A2,A0 02#A1,A2,A0 +thwart 01#A1,A0 +tick 01#A0 02#A1,A0 03#A1,A0 +ticket 01#A1,A2,A0 +tie 01#A1,A2,A3,A0 02#A1,A0 03#A1,A0 04#A1,A0 05#A1,A2 06#A1,A2 +tighten 01#A1,A0 02#A1,A0 +tilt 01#A1,A2,A0 +time 01#A1,A2,A0 02#A1,A2,A0 +tinker 01#A1,A0 +tip 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 04#A1,A0 +tiptoe 01#A0 +tire 01#A1,A0 02#A1,A0 03#A1,A0 +title 01#A1,A2,A0 +toast 01#A1,A0 +toil 01#A1,A0 +tolerate 01#A1,A0 +tone 01#A1,A0 +toot 01#A1,A0 +top 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 +topple 01#A1,A2,A3,A4,A0 +torch 01#A1,A0 +torment 01#A1,A2,A0 +torpedo 01#A1,A0 +torture 01#A1,A2,A0 +toss 01#A1,A0 02#A1,A0 +tot 01#A1,A2,A3,A4,A0 +total 01#A1,A2 02#A1,A0 03#A1,A0 +tote 01#A1,A2,A3,A4,A0 02#A1,A0 +totter 01#A1 02#A1 +touch 01#A1,A2,A0 02#A1,A2,A3,A0 03#A1,A2,A0 04#A1,A0 +toughen 01#A1,A0 +tour 01#A1,A0 +tout 01#A1,A2,A3,A0 +tower 01#A1,A0 +toy 01#A1,A0 +trace 01#A1,A2,A0 02#A1,A2,A0 +track 01#A1,A2,A3,A0 02#A1,A2,A0 03#A1,A2 +trade 01#A1,A2,A3,A4,A0 +traduce 01#A1,A0 +trail 01#A1,A0 02#A1 +train 01#A1,A2,A0 02#A1,A2,A0 +traipse 01#A1,A0 +tramp 01#A1,A0 +trample 01#A1,A0 +transact 01#A1,A0 +transcribe 01#A1,A0 +transfer 01#A1,A2,A3,A0 +transform 01#A1,A2,A3,A0 +translate 01#A1,A2,A3,A0 +transmit 01#A1,A2,A0 +transmogrify 01#A1,A2,A3,A0 +transplant 01#A1,A2,A3,A0 +transport 01#A1,A2,A3,A0 +trap 01#A1,A2,A0 +trash 01#A1,A0 +traumatize 01#A1,A2,A0 +travel 01#A1,A0 +tread 01#A0 +treat 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A3,A0 04#A1,A2,A0 05#A1,A0 +treble 01#A1,A0 +trek 01#A1,A0 +tremble 01#A1 +trend 01#A1,A2 +trespass 01#A1,A0 +trick 01#A1,A0 +trickle 01#A1,A2,A0 +trigger 01#A1,A2,A3,A0 +trim 01#A1,A2,A3,A0 02#A1,A2,A3,A4,A0 03#A1,A2,A0 +trip 01#A1,A0 +triple 01#A1,A2,A3,A4,A0 +triumph 01#A1,A2,A0 +trivialize 01#A1,A0 +trot 01#AA,A1,A0 02#A1,A0 +trouble 01#A1,A2,A0 +trudge 01#A1,A0 +trump 01#A1,A0 +trumpet 01#A1,A2,A0 +trundle 01#A1,A0 +trust 01#A1,A2,A0 02#A1,A0 +try 01#A1,A0 02#A1,A0 03#A1,A2,A0 04#A1,A2,A0 +tuck 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 +tug 01#A1,A2,A0 02#A1,A0 03#A1,A2,A0 +tumble 01#A1,A2,A3,A4,AM,A0 +tune 01#A1,A2,A0 02#A1,A2,A0 +turn 11#A1,A2 01#A1,Am,A0 12#A1,A2,A3,A0 02#A1,A2,A3,A0 13#A1,A0 03#A1,A2,A0 14#A1,A0 04#A1,A2,A0 15#A1,A0 16#A1,A0 05#A1,A0 06#A1,A2,A0 17#A1 07#A1,A0 08#A1,A2,A0 09#A1,A0 10#A1,A2,A3,A0 +tutor 01#A1,A2,A0 +twiddle 01#A1,A0 +twin 01#A1,A0 +twist 01#A1,A0 +twitch 01#A1,A0 +type 01#A1,A0 02#A1,A0 +typify 01#A1,A0 +unblock 01#A1,A0 +uncover 01#A1,A3,A0 +undercut 01#A1,A0 +underestimate 01#A1,A2,A0 +undergo 01#A1,A2 +underlie 01#A1,A0 +underline 01#A1,A0 +undermine 01#A1,A0 +underperform 01#A1,A0 +underpin 01#A1,A0 +underprice 01#A1,A0 +underscore 01#A1,A2,A0 +undersell 01#A1,A0 +understand 01#A1,A2,A0 +understate 01#A1,A2,A0 +undertake 01#A1,A2,A0 +undervalue 01#A1,A2,A0 +underwrite 01#A1,A0 +undo 01#A1,A0 +undulate 01#A1,A0 +unearth 01#A1,A0 +unfold 01#A1,A0 02#A1 +unhinge 01#A1,A0 02#A1,A2,A0 +unify 01#A1,A2,A3,A0 +unionize 01#A1,A0 +unite 01#A1,A2,A3,A0 +unleash 01#A1,A0 +unload 01#A1,A2,A0 02#A1,A2,A3,A0 +unlock 01#A1,A2,A0 +unmask 01#A1,A0 +unnerve 01#A1,A2,A0 +unplug 01#A1,A0 +unravel 01#A1,A0 +unroll 01#A1,A0 +unsettle 01#A1,A2,A0 +unveil 01#A1,A2,A0 +unwind 01#A1,A0 02#A1,A2 +up 01#A1,A0 +update 01#A1,A0 +upgrade 01#A1,A2,A0 02#A1,A2,A3,A4,A0 +uphold 01#A1,A0 +uproot 01#A1,A0 +upset 01#A1,A2,A0 02#A1,A0 +urge 01#A1,A2,A0 +use 01#A1,A2,A0 02#A1,A0 +usher 01#A1,A0 02#A1,A2,A0 +usurp 01#A1,A0 +utilize 01#A1,A0 +utter 01#A1,A0 +vacate 01#A1,A0 +vacation 01#A1,A0 +vacillate 01#A1,A0 +vacuum 01#A1,A2,A0 +validate 01#A1,A0 +value 01#A1,A2,A0 +vanish 01#A1 +vary 01#A1,A2,A3,A4,A5,A0 +vault 01#A1,A2,A3,A4,A0 +veer 01#A1,A0 +veil 01#A1,A0 +vend 01#A1,A0 +vent 01#A1,A0 +ventilate 01#A1,A0 +venture 01#A0 02#A1,A0 +verge 01#A1,A2 +verify 01#A1,A0 +vest 01#A1,A2,A0 +veto 01#A1,A2,A0 +vex 01#A1,A0 +vibrate 01#A1,A0 +victimize 01#A1,A2,A0 +vie 01#A1,A2,A0 +view 01#A1,A0 02#A1,A2,A0 +vindicate 01#A1,A2,A0 +violate 01#A1,A0 +visit 01#A1,A0 +visualize 01#A1,A2,A0 +vitiate 01#A1,A0 +voice 01#A1,A2,A0 +void 01#A1,A0 +volunteer 01#A1,A2,A0 +vomit 01#A0 02#A1,A0 +vote 01#AA,A1,A2,A0 02#A1,A2,A0 03#A1,A0 +vow 01#A1,A0 +wad 01#A1,A2,A0 +waddle 01#A0 +wade 01#A1,A0 +waffle 01#A1,A0 +waft 01#A1,A0 +wag 01#A1,A0 +wage 01#A1,A2,A0 +wail 01#A1,A0 +wait 01#A1,A2,Am 02#A1,A0 03#A1,A0 +waive 01#A1,A2,A0 +wake 01#A1,A2,A0 02#A1,A2,A0 +walk 01#AA,A1,A0 02#A1,A0 03#A1,A2,A0 +wall 01#A1,A2,A0 +wallow 01#A1,A0 +wander 01#A1,A0 +wane 01#A1 +want 01#A1,A2,A3,A4,A0 +war 01#A1,A0 +ward 01#A1,A2,A0 +warm 01#A1,A0 02#A1,A0 03#A1,A0 04#A1,A0 +warn 01#A1,A2,A0 +warrant 01#A1,A0 +wash 01#A1,A2,A3,A4,A0 02#A0 +waste 01#A1,A2,A0 +watch 01#A1,A0 02#A1,A0 +water 01#A1,A0 02#A1,A0 +wave 01#A1,A2,A0 02#A1,A2,A0 +waver 01#A1,A0 +wax 01#A1,A2,A0 02#A1 03#A1,A2 +weaken 01#A1,A2,A3,A4,A0 +wean 01#A1,A2,A0 +wear 01#A1,A0 02#A1,A0 03#A1,A0 04#A1 +weather 01#A1,A0 +weave 01#A1,A2,A0 02#A1,A0 +wed 01#A1,A2,A0 +wedge 01#A1,A2,A0 +weep 01#A1,A0 +weigh 01#A1,A2,A3,A0 02#A1,A0 03#A1,A0 04#A1,A2,A0 +weight 01#A1,A2,A0 02#A1,A0 +welcome 01#A1,A0 +weld 01#A1,A0 +whack 01#A1,A2,A0 02#A1,A2,A0 +wheel 01#A1,A0 +wheeze 01#A1,A2,A0 02#A0 +whip 01#A1,A2,A0 02#A1,A0 03#A1,A0 +whipsaw 01#A1,A0 +whirl 01#A1,A2 +whirr 01#A0 +whisk 01#A1,A2,A0 +whisper 01#A1,A2,A0 +whistle 01#A1,A2,A0 +whittle 01#A1,A2,A4,A0 02#A1,A2,A3,A4,A0 +wholesale 01#A1,A2,A3,A4,A0 +whoop 01#A1,A2,A0 +whoosh 01#A0 02#A0 +widen 01#A1,A2,A3,A4,A0 +widow 01#A1,A0 +wield 01#A1,A2,A3,A0 +wiggle 01#A1,A0 02#A1,A0 +will 01#A1,A2,A0 +win 01#A1,A2,A3,A4,A0 02#A1,A2,A0 +wind 01#A1,A2,A0 02#A1,A2 03#A1,A2,A0 04#A1,A0 +wink 01#A1,A2,A0 +wipe 01#A1,A2,A3,A0 02#A1,A2,A0 +wire 01#A1,A2,A0 02#A1,A2,A0 +wish 01#A1,A2,A0 +withdraw 01#A1,A2,A0 +wither 01#A1,A0 +withhold 01#A1,A2,A0 +withstand 01#A1,A0 +witness 01#A1,A0 +womanize 01#A0 +wonder 01#A1,A0 02#A1,A0 +woo 01#A1,A0 +word 01#A1,A0 +work 01#A1,A2,A3,A4,A0 02#A1,A2,A0 03#A0 04#A1,A2,A0 05#A1,A2,A0 06#A1,A3,A4,A0 07#A1,A2,A0 08#A1,A2,A0 +worry 01#A1,A2,A0 02#A1,A0 +worsen 01#A1,A2,A0 +wound 01#A1,A2,A0 +wow 01#A1,A2,A0 +wrack 01#A1,A0 +wrangle 01#A1,A2,A0 +wrap 01#A1,A2,A0 02#A1,A2,A0 +wreak 01#A1,A2,A0 +wreck 01#A1,A2,A0 +wrench 01#A1,A2,A0 +wrest 01#A1,A2,A0 +wrestle 01#A1,A2,A0 02#A1,A2,A0 03#A1,A2,A0 +wriggle 01#A1,A0 02#A1,A0 +wring 01#A1,A0 02#A1,A2,A0 03#A1,A0 +write 01#A1,A2,A0 02#A1,A2,A0 03#A1,A0 04#A1,A2,A3,A4,A0 05#A1,A0 +writhe 01#A1 +wrong 01#A1,A0 +yank 01#A1,A2,A0 +yell 01#A1,A2,A0 +yelp 01#A1,A2,A0 +yield 01#A1,A0 02#A1,A2,A0 03#A1,A2,A0 +zap 01#A1,A0 +zero 01#A1,A0 02#A1,A0 +zip 01#A0 +zone 01#A1,A2,A0 +zoom 01#A1,A2,A3,A4,AM diff --git a/srl/src/main/resources/constraints/Nom.constraints b/srl/src/main/resources/constraints/Nom.constraints new file mode 100644 index 000000000..257e53ccc --- /dev/null +++ b/srl/src/main/resources/constraints/Nom.constraints @@ -0,0 +1,14 @@ +noOverlappingArguments +noDuplicateCore +CArgumentConstraint +; RArgumentConstraint +predicateSense + +;; It is disappointing that this constraint doesn't improve +;; performance. I think this is because of noise in the data. + +; supportVerbConstraint + +noCrossArgumentExclusiveOverlap + +crossArgumentRetainedModifiers diff --git a/srl/src/main/resources/constraints/Verb.constraints b/srl/src/main/resources/constraints/Verb.constraints new file mode 100644 index 000000000..9509f54d9 --- /dev/null +++ b/srl/src/main/resources/constraints/Verb.constraints @@ -0,0 +1,10 @@ +noOverlappingArguments +noDuplicateCore +CArgumentConstraint +RArgumentConstraint +predicateSense +beVerbConstraint +noCrossArgumentExclusiveOverlap +crossArgumentRetainedModifiers + +; atLeastOneCoreArgument ; Bad constraint diff --git a/srl/src/main/resources/features/Nom.Classifier.fex b/srl/src/main/resources/features/Nom.Classifier.fex new file mode 100644 index 000000000..1ac5f5c41 --- /dev/null +++ b/srl/src/main/resources/features/Nom.Classifier.fex @@ -0,0 +1,63 @@ +(define features + + (define word-context + (context :size 2 + :include-index true + :ignore-center true + word)) + + (define pos-context + (context :size 2 + :include-index true + :ignore-center true + pos)) + + (define predicate-features + (list word + pos + lemma + capitalization + + word-context + pos-context + + (attribute :name predicate) + + (subcat-frame :parse-view *default-parser*) + nom-lex-class)) + + (define arg-features + (list (transform-input constituent-parent + predicate-features) + (head-features :parse-view *default-parser* + (list word + pos + numbers + gerunds + + nom + days-of-week + + months + date + wn:synsets-first-sense + wn:hypernyms-first-sense)) + currency + linear-position + + hyphen-argument-feature + + (parse-phrase-features :parse-view *default-parser*) + + (parse-path :parse-view *default-parser*) + chunk-embedding + chunk-path + ne-embedding + + (parse-siblings :parse-view *default-parser*) + word-context + pos-context)) + + (list (conjoin arg-features + arg-features))) + diff --git a/srl/src/main/resources/features/Nom.Identifier.fex b/srl/src/main/resources/features/Nom.Identifier.fex new file mode 100644 index 000000000..4cf084bf8 --- /dev/null +++ b/srl/src/main/resources/features/Nom.Identifier.fex @@ -0,0 +1,65 @@ +(define features + + (define word-context + (context :size 2 + :include-index true + :ignore-center true + word)) + + (define pos-context + (context :size 2 + :include-index true + :ignore-center true + pos)) + + ;; The predicate features are turned off for now. The question is + ;; whether they are actually needed at all if no feature + ;; conjunctions are used. + + (define predicate-features + (transform-input constituent-parent + (list word + pos + lemma + capitalization + + (attribute :name predicate) + + (subcat-frame :parse-view *default-parser*) + nom-lex-class))) + + (define head-word-features + (head-features :parse-view *default-parser* + (list word + pos + numbers + gerunds + + nom + days-of-week + + months + date))) + + + (list head-word-features + linear-position + (parse-phrase-features :parse-view *default-parser*) + (parse-path :parse-view *default-parser*) + hyphen-argument-feature + + (conjoin predicate-features + (list head-word-features + (parse-path :parse-view *default-parser*) + linear-position + (parse-phrase-features :parse-view *default-parser*))) + + + chunk-embedding + chunk-path + ne-embedding + currency + + (parse-siblings :parse-view *default-parser*) + word-context + pos-context)) diff --git a/srl/src/main/resources/features/Nom.Predicate.fex b/srl/src/main/resources/features/Nom.Predicate.fex new file mode 100644 index 000000000..ab1446c54 --- /dev/null +++ b/srl/src/main/resources/features/Nom.Predicate.fex @@ -0,0 +1,31 @@ +(define features + + (define word-context + (context :size 2 + :include-index true + :ignore-center true + word)) + + (define pos-context + (context :size 2 + :include-index true + :ignore-center true + pos)) + + (define predicate-features + (list word-context + pos-context + word + lemma + pos + capitalization + (attribute :name predicate) + (subcat-frame :parse-view *default-parser*) + numbers + chunk-embedding + ne-embedding)) + + (list predicate-features + (conjoin nom-lex-class + predicate-features))) + diff --git a/srl/src/main/resources/features/Nom.Sense.fex b/srl/src/main/resources/features/Nom.Sense.fex new file mode 100644 index 000000000..0b0a5f4c4 --- /dev/null +++ b/srl/src/main/resources/features/Nom.Sense.fex @@ -0,0 +1,49 @@ +(define features + + (define word-features + (list word + pos + lemma + capitalization + conflated-pos + gerunds + nom + ne-embedding + brown-clusters-1000 + de-verbal-suffixes + de-nom-nouns + de-adj-nouns + known-prefixes + wn:exists-entry + wn:synsets-first-sense + wn:synsets-all + wn:hypernyms-all + wn:hypernyms-first-sense + wn:part-holonyms-first-sense + wn:member-holonyms-first-sense + wn:substance-holonyms-first-sense + ;; gazetteers + )) + (define my-features + (list (context :size 3 + :include-index true + :ignore-center true + word + pos + lemma + has-modal-verb + is-negated) + (context :size 1 + :include-index true + :ignore-center true + (bigram word) + (trigram word)) + (governor-features :parse-view *default-parser* + word-features) + (object-features :parse-view *default-parser* + word-features))) + (list my-features + (conjoin lemma my-features))) + + + diff --git a/srl/src/main/resources/features/Verb.Classifier.fex b/srl/src/main/resources/features/Verb.Classifier.fex new file mode 100644 index 000000000..b32cb65a0 --- /dev/null +++ b/srl/src/main/resources/features/Verb.Classifier.fex @@ -0,0 +1,71 @@ +(define features + + (define predicate-features + (list (attribute :name predicate) + pos + (voice :parse-view *default-parser*) + (subcat-frame :parse-view *default-parser*) + has-modal-verb + is-negated + (parse-phrase-features :parse-view *default-parser*) + + (context :size 1 + :include-index true + :ignore-center false + (list word + pos + (conjoin word pos))) + + levin-verb-class)) + + (define word-pos + (list word pos)) + + (define parse-sibling-features + (list label + (head-features :parse-view *default-parser* + word-pos))) + + (define arg-features + (list (transform-input constituent-parent + predicate-features) + (parse-phrase-features :parse-view *default-parser*) + linear-position + (parse-path :parse-view *default-parser*) + (context :size 2 + :include-index true + :ignore-center true + word-pos) + + (transform-input first-word + word-pos) + + (transform-input last-word + word-pos) + + ;; The next three are from Surdeanu et al. + (transform-input parse-left-sibling + parse-sibling-features) + + (transform-input parse-right-sibling + parse-sibling-features) + + pp-features + + projected-path ;; from Toutanova etal + + + ne-embedding + + length + chunk-embedding + chunk-path + (clauses :parse-view *default-parser*) + + (syntactic-frame :parse-view *default-parser*) + + (head-features :parse-view *default-parser* + word-pos))) + (conjoin arg-features + arg-features)) + diff --git a/srl/src/main/resources/features/Verb.Identifier.fex b/srl/src/main/resources/features/Verb.Identifier.fex new file mode 100644 index 000000000..b45e6bf28 --- /dev/null +++ b/srl/src/main/resources/features/Verb.Identifier.fex @@ -0,0 +1,78 @@ +(define features + + (define predicate-features + (list (attribute :name predicate) + pos + (voice :parse-view *default-parser*) + (subcat-frame :parse-view *default-parser*) + has-modal-verb + is-negated + (parse-phrase-features :parse-view *default-parser*) + + (context :size 1 + :include-index true + :ignore-center false + (list word + pos + (conjoin word pos))) + + levin-verb-class)) + + (define word-pos + (list word pos)) + + (define parse-sibling-features + (list label + (head-features :parse-view *default-parser* + word-pos))) + + (define arg-features + (list (transform-input constituent-parent + predicate-features) + (parse-phrase-features :parse-view *default-parser*) + linear-position + (parse-path :parse-view *default-parser*) + (context :size 2 + :include-index true + :ignore-center true + word-pos) + + (transform-input first-word + word-pos) + + (transform-input last-word + word-pos) + + ;; The next three are from Surdeanu et al. + (transform-input parse-left-sibling + parse-sibling-features) + + (transform-input parse-right-sibling + parse-sibling-features) + + pp-features + + projected-path ;; from Toutanova etal + + + ne-embedding + + ;; Only specific conjunctions here + (conjoin predicate-features + (list (parse-path :parse-view *default-parser*) + (parse-phrase-features :parse-view *default-parser*) + linear-position + pp-features)) + + + length + chunk-embedding + chunk-path + (clauses :parse-view *default-parser*) + + (syntactic-frame :parse-view *default-parser*) + + (head-features :parse-view *default-parser* + word-pos))) + + arg-features) diff --git a/srl/src/main/resources/features/Verb.Predicate.fex b/srl/src/main/resources/features/Verb.Predicate.fex new file mode 100644 index 000000000..19af471ff --- /dev/null +++ b/srl/src/main/resources/features/Verb.Predicate.fex @@ -0,0 +1,30 @@ +(define features + + (define word-context + (context :size 2 + :include-index true + :ignore-center true + word)) + + (define pos-context + (context :size 2 + :include-index true + :ignore-center true + pos)) + + (define predicate-features + (list word-context + pos-context + word + lemma + pos + capitalization + (attribute :name predicate) + (subcat-frame :parse-view *default-parser*) + numbers + chunk-embedding + ne-embedding)) + + predicate-features) + + diff --git a/srl/src/main/resources/features/Verb.Sense.fex b/srl/src/main/resources/features/Verb.Sense.fex new file mode 100644 index 000000000..548013408 --- /dev/null +++ b/srl/src/main/resources/features/Verb.Sense.fex @@ -0,0 +1,49 @@ +(define features + + (define word-features + (list word + pos + lemma + capitalization + conflated-pos + gerunds + nom + ne-embedding + brown-clusters-1000 + de-verbal-suffixes + de-nom-nouns + de-adj-nouns + known-prefixes + wn:exists-entry + wn:synsets-first-sense + wn:synsets-all + wn:hypernyms-all + wn:hypernyms-first-sense + wn:part-holonyms-first-sense + wn:member-holonyms-first-sense + wn:substance-holonyms-first-sense + ;; gazetteers + )) + (define my-features + (list (context :size 3 + :include-index true + :ignore-center true + word + pos + lemma + has-modal-verb + is-negated) + (context :size 1 + :include-index true + :ignore-center true + (bigram word) + (trigram word)) + (governor-features :parse-view *default-parser* + word-features) + (object-features :parse-view *default-parser* + word-features))) + (list my-features + (conjoin lemma my-features))) + + + diff --git a/srl/src/main/resources/log4j.properties b/srl/src/main/resources/log4j.properties new file mode 100644 index 000000000..ea57ac43b --- /dev/null +++ b/srl/src/main/resources/log4j.properties @@ -0,0 +1,8 @@ +# Root logger option +log4j.rootLogger=INFO, stdout + +# Direct log messages to stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{HH:mm:ss} %-5p %c{1}:%L - %m%n \ No newline at end of file diff --git a/srl/src/main/resources/nombank.list.gz b/srl/src/main/resources/nombank.list.gz new file mode 100644 index 000000000..b90a3ca5c Binary files /dev/null and b/srl/src/main/resources/nombank.list.gz differ diff --git a/srl/src/test/java/edu/illinois/cs/cogcomp/srl/NonJUnitMemoryUseTest.java b/srl/src/test/java/edu/illinois/cs/cogcomp/srl/NonJUnitMemoryUseTest.java new file mode 100644 index 000000000..e31bf6ffc --- /dev/null +++ b/srl/src/test/java/edu/illinois/cs/cogcomp/srl/NonJUnitMemoryUseTest.java @@ -0,0 +1,156 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl; + +import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.utilities.DummyTextAnnotationGenerator; +import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager; +import edu.illinois.cs.cogcomp.edison.annotators.ClauseViewGenerator; +import edu.illinois.cs.cogcomp.srl.config.SrlConfigurator; +import edu.illinois.cs.cogcomp.srl.core.SRLType; + +import java.util.Properties; + +/** + * Attempt to determine whether the lazy initialization Annotator API from i-c-u creates memory use + * problems + * + * @author mssammon + */ +public class NonJUnitMemoryUseTest { + + private static final String CONFIG = "src/test/resources/srl-config.properties"; + + private static String[] requiredViews = new String[] {ViewNames.POS, ViewNames.LEMMA, + ViewNames.SHALLOW_PARSE, ViewNames.PARSE_STANFORD, ViewNames.NER_CONLL}; + + private ResourceManager rm; + + static SemanticRoleLabeler srlStatic; + + + public static void main(String[] args) { + + NonJUnitMemoryUseTest test = new NonJUnitMemoryUseTest(); + + try { + test.setUp(); + } catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } + + boolean isStatic = false; + System.out.println("Test method-scope SRL:"); + + try { + test.testVerbSRL(isStatic); + } catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } + + System.out.println("Finished first test. Running again."); + + try { + test.testVerbSRL(isStatic); + } catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } + + System.out.println("Running test again, this time with static field."); + + isStatic = true; + + try { + test.testVerbSRL(isStatic); + } catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } + + System.out.println("setting static field to null, and rerunning..."); + srlStatic = null; + showMemoryUsage(); + try { + test.testVerbSRL(isStatic); + } catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } + + + } + + + public void setUp() throws Exception { + ResourceManager tempRm = new ResourceManager(CONFIG); + Properties props = new Properties(); + props.setProperty(SrlConfigurator.INSTANTIATE_PREPROCESSOR.key, SrlConfigurator.TRUE); + rm = new ResourceManager(props); + rm = SrlConfigurator.mergeProperties(tempRm, rm); + } + + public void testVerbSRL(boolean isStatic) throws Exception { + + TextAnnotation ta = + DummyTextAnnotationGenerator.generateAnnotatedTextAnnotation(requiredViews, false, + 1); + // if (!ta.hasView(ViewNames.CLAUSES_STANFORD)) // an additional "invisible" dependency + // ta.addView(ClauseViewGenerator.STANFORD); + SemanticRoleLabeler verbSRL = getSrl(isStatic); + PredicateArgumentView srl = (PredicateArgumentView) verbSRL.getView(ta); + + System.out.println("SRL output: " + srl.toString()); + System.out.println("memory use" + (isStatic ? "before srl goes out of scope" : "") + ":"); + showMemoryUsage(); + } + + private SemanticRoleLabeler getSrl(boolean isStatic) throws Exception { + Properties props = new Properties(); + props.setProperty(SrlConfigurator.SRL_TYPE.key, SRLType.Verb.name()); + + rm = SrlConfigurator.mergeProperties(rm, new ResourceManager(props)); + + SemanticRoleLabeler verbSRL = null; + + if (!isStatic || null == srlStatic) + verbSRL = new SemanticRoleLabeler(rm, false); + + if (isStatic && null == srlStatic) + srlStatic = verbSRL; + + return verbSRL; + } + + + public static void showMemoryUsage() { + int mb = 1024 * 1024; + + // Getting the runtime reference from system + Runtime runtime = Runtime.getRuntime(); + + System.out.println("##### Heap utilization statistics [MB] #####"); + + // Print used memory + System.out.println("Used Memory:" + (runtime.totalMemory() - runtime.freeMemory()) / mb); + + // Print free memory + System.out.println("Free Memory:" + runtime.freeMemory() / mb); + + // Print total available memory + System.out.println("Total Memory:" + runtime.totalMemory() / mb); + + // Print Maximum available memory + System.out.println("Max Memory:" + runtime.maxMemory() / mb); + } + +} diff --git a/srl/src/test/java/edu/illinois/cs/cogcomp/srl/SemanticRoleLabelerTest.java b/srl/src/test/java/edu/illinois/cs/cogcomp/srl/SemanticRoleLabelerTest.java new file mode 100644 index 000000000..7a75d567f --- /dev/null +++ b/srl/src/test/java/edu/illinois/cs/cogcomp/srl/SemanticRoleLabelerTest.java @@ -0,0 +1,90 @@ +/** + * This software is released under the University of Illinois/Research and Academic Use License. See + * the LICENSE file in the root folder for details. Copyright (c) 2016 + * + * Developed by: The Cognitive Computation Group University of Illinois at Urbana-Champaign + * http://cogcomp.cs.illinois.edu/ + */ +package edu.illinois.cs.cogcomp.srl; + +import edu.illinois.cs.cogcomp.core.datastructures.ViewNames; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.PredicateArgumentView; +import edu.illinois.cs.cogcomp.core.datastructures.textannotation.TextAnnotation; +import edu.illinois.cs.cogcomp.core.utilities.DummyTextAnnotationGenerator; +import edu.illinois.cs.cogcomp.core.utilities.configuration.ResourceManager; +import edu.illinois.cs.cogcomp.edison.annotators.ClauseViewGenerator; +import edu.illinois.cs.cogcomp.srl.config.SrlConfigurator; +import edu.illinois.cs.cogcomp.srl.core.SRLType; +import org.junit.*; + +import java.util.Properties; + +public class SemanticRoleLabelerTest { + + private static final String CONFIG = "src/test/resources/srl-config.properties"; + + private static String[] requiredViews = new String[] {ViewNames.POS, ViewNames.LEMMA, + ViewNames.SHALLOW_PARSE, ViewNames.PARSE_STANFORD, ViewNames.NER_CONLL}; + + private ResourceManager rm; + + @Before + public void setUp() throws Exception { + + // at least 8GB memory required to run these tests + Assume.assumeTrue(Runtime.getRuntime().maxMemory() / Math.pow(10, 9) > 8.0); + + ResourceManager tempRm = new ResourceManager(CONFIG); + Properties props = new Properties(); + props.setProperty(SrlConfigurator.INSTANTIATE_PREPROCESSOR.key, SrlConfigurator.TRUE); + rm = new ResourceManager(props); + rm = SrlConfigurator.mergeProperties(tempRm, rm); + + } + + @Test + public void testVerbSRL() throws Exception { + Properties props = new Properties(); + props.setProperty(SrlConfigurator.SRL_TYPE.key, SRLType.Verb.name()); + + rm = SrlConfigurator.mergeProperties(rm, new ResourceManager(props)); + + SemanticRoleLabeler verbSRL = new SemanticRoleLabeler(rm); + TextAnnotation ta = + DummyTextAnnotationGenerator.generateAnnotatedTextAnnotation(requiredViews, false, + 1); + if (!ta.hasView(ViewNames.CLAUSES_STANFORD)) // an additional "invisible" dependency + ta.addView(ClauseViewGenerator.STANFORD); + PredicateArgumentView srl = (PredicateArgumentView) verbSRL.getView(ta); + + String expected = + "finish:02\n A1: The construction of the John Smith library\n AM-MNR: on time\n" + + "design:01\n A1: The $10M building\n AM-MNR: designed\n AM-TMP: in 2016\n" + + "paving:01\ncommence:01\n A0: The paving\n A1: Monday\n" + + "finish:02\n A1: finish\n AM-MOD: will\n AM-TMP: in June\n"; + // "finish:02\n A1: The construction of the library\n AM-TMP: on time\n" + Assert.assertEquals(expected, srl.toString()); + } + + + /** + * TODO: why does nom only annotate the first sentence? no predicates in other sentences? + * + * @throws Exception + */ + @Test + public void testNomSRL() throws Exception { + Properties props = new Properties(); + props.setProperty(SrlConfigurator.SRL_TYPE.key, SRLType.Nom.name()); + rm = SrlConfigurator.mergeProperties(rm, new ResourceManager(props)); + + SemanticRoleLabeler nomSRL = new SemanticRoleLabeler(rm); + TextAnnotation ta = + DummyTextAnnotationGenerator.generateAnnotatedTextAnnotation(requiredViews, false, + 1); + PredicateArgumentView srl = (PredicateArgumentView) nomSRL.getView(ta); + + String expected = "construction:01\n A1: of the John Smith library\nlibrary:01\n"; + Assert.assertEquals(expected, srl.toString()); + } +} diff --git a/srl/src/test/resources/srl-config.properties b/srl/src/test/resources/srl-config.properties new file mode 100644 index 000000000..1a052a39d --- /dev/null +++ b/srl/src/test/resources/srl-config.properties @@ -0,0 +1,39 @@ +## Illinois SRL Configuration## + +# Whether to use the Illinois Curator to get the required annotations for training/testing +# If set to false, Illinois NLP pipeline will be used +UseCurator = false + +# The configuration of the Illinois NLP pipeline +PipelineConfig = config/pipeline.properties + +# The parser used to extract constituents and syntactic features +# Options are: Charniak, Berkeley, Stanford +# NB: Only Stanford can be used in standalone mode. +DefaultParser = Stanford + +# The configuration for the Structured learner +LearnerConfig = config/learner.properties + +# Num of threads for feat. ext. +NumFeatExtThreads = 10 + +# The ILP solver to use for the joint inference +# Options are: Gurobi, OJAlgo +ILPSolver = OJAlgo + +### Training corpora directories ### +# This is the directory of the merged (mrg) WSJ files +PennTreebankHome = /shared/corpora/corporaWeb/treebanks/eng/pennTreebank/treebank-3/parsed/mrg/wsj/ +PropbankHome = /shared/corpora/corporaWeb/treebanks/eng/propbank_1/data +NombankHome = /shared/corpora/corporaWeb/treebanks/eng/nombank/ + +# The directory of the sentence and pre-extracted features database (~5G of space required) +# Not used during test/working with pre-trained models +CacheDirectory = cache + +ModelsDirectory = models + +# Directory to output gold and predicted files for manual comparison +# Comment out for no output +OutputDirectory = srl-out \ No newline at end of file