Add initial core code

Author: stsnel

Diff for: README.md

@@ -1,2 +1,29 @@
 # python-string-extractor
+
 Extracts strings relevant to control flow from Python code
+
+This package extracts strings (including prefixes, suffixes and fragments)
+from Python code that seems (potentially) relevant to control flow. Having
+a list of strings that are potentially relevant to the execution path can
+help with scriptless testing or automated test case generation.
+
+For, example in this code example strings "foo" and "bar" are relevant to
+the control flow, so it might be useful to include them in test input for
+the software.
+
+```
+if var == "foo":
+   doSomething()
+elif var == "bar":
+   doSomethingElse()
+```
+
+## Usage
+
+The string extractor needs to be provided with an execution trace of Python
+software. This trace can be obtained using a trace function, or a modified
+version of coverage measurement software.
+
+The string extractor has a cache, which is enabled by default and can
+optionally be persisted to a file. This helps with efficiently handling loops
+in code, as well as overlapping execution traces.

Diff for: setup.py

@@ -0,0 +1,11 @@
+from setuptools import setup
+
+setup(
+    author="Sietse Snel",
+    author_email="[email protected]",
+    description=('Extracts strings relevant to control flow from python code'),
+    install_requires=[],
+    name='string_extractor',
+    packages=['string_extractor'],
+    version='0.0.1'
+)

Diff for: string_extractor/__init__.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+import ast
+import linecache
+import re
+
+from string_extractor.string_collector import InterestingStringCollector
+
+class StringExtractor:
+    def _preprocessLine(self, filename, lineNumber):
+        """ Retrieves a line of code for parsing, checks whether it can be parsed
+            and performs any transformations needed for dealing with control structures
+            or multiline statements.
+            
+            Result is a 2-tuple, with the first element being a status code, and the second
+            element being the preprocessed statement to be parsed.
+            
+            Status codes:
+              OK           : line can be parsed
+              NOTSUPPORTED : the parser should ignore this line, because the preprocessor
+                             doesn't support this particular type of statement (in this context).
+              ERROR        : the preprocessor was unable to transform the line into a
+                             parsable form. This could happen for long multiline statements (length
+                             more than maxMultilineLength), or for language constructs that aren't
+                             recognized by the preprocessor.
+            """
+        maxMultilineLength= 20
+        numberOfLines = self._getNumberOfLines(filename)
+        line = linecache.getline(filename, lineNumber).strip()
+
+        if filename.endswith(".j2"):
+            return ("IGNORE","")
+
+        # Ignore irrelevant control flow statements
+        for keyword in ["try", "except", "finally", "else" ]:
+            if re.match( "^{}\s*[\s:]$".format(keyword), line):
+                return ("IGNORE", "")
+            elif re.match( "^{}\s".format(keyword), line):
+                return ("IGNORE", "")
+
+        # Normalize elif statement to if statement.
+        if re.match( "^elif\s", line):
+            line = line[2:]
+
+        # Check whether we are dealing with a relevant compound statement
+        compoundStatement = False
+        for keyword in ["for", "if", "while"]:
+            if re.match( "^{}[\(\s]".format(keyword), line):
+                compoundStatement = True
+
+        statement = ""
+        max_offset = max ( maxMultilineLength, numberOfLines - lineNumber )
+        for offset in range(numberOfLines - lineNumber):
+            thisLineNumber = lineNumber + offset
+
+            if offset == 0:
+                thisLine = line
+            else:
+                thisLine = linecache.getline(filename, thisLineNumber).strip()
+
+            statement = " ".join([statement, thisLine ]).strip()
+
+            if statement.endswith("\\"):
+                # Explicit line continuation. Keep iterating over lines,
+                # even if maximum statement length has been exceeded.
+                statement = statement[:-1]
+                continue
+            elif compoundStatement and statement.endswith(":"):
+                # It seems we have a complete control structure statement.
+                # Add a dummy block.
+                statementWithDummyBlock = statement + "\n  pass"
+                if self._isParsable(statementWithDummyBlock):
+                    return ("OK", statementWithDummyBlock)
+
+            elif (not compoundStatement) and self._isParsable(statement):
+                return ("OK", statement)
+
+            if offset >= maxMultilineLength:
+                break
+
+        return ("ERROR", "")
+
+
+    def _isParsable(self, statement):
+        """Determines whether a statement is parsable."""
+        try:
+            tree = ast.parse(statement)
+            return True
+        except SyntaxError:
+            return False
+
+
+    def _getNumberOfLines(self, filename):
+        with open(filename, 'r') as file:
+            for count, line in enumerate(file):
+                pass
+        return count + 1
+
+    def getInterestingStrings(self, statement):
+        tree = ast.parse(statement)
+        collector = InterestingStringCollector()
+        collector.visit(tree)
+        return collector.getCollectedStrings()
+

Diff for: string_extractor/string_collector.py

@@ -0,0 +1,50 @@
+""" Internal class for extracting relevant strings from
+    an ast parse tree, as a visitor."""
+
+import ast
+
+class InterestingStringCollector(ast.NodeVisitor):
+    def __init__(self):
+        self.suffixes = set()
+        self.prefixes = set()
+        self.fragments = set()
+        self.fullStrings = set()
+
+
+    def getCollectedStrings(self):
+        return ( [ ( "SUFFIX", s ) for s in self.suffixes ] +
+                 [ ( "PREFIX", s ) for s in self.prefixes ] +
+                 [ ( "FRAGMENT", s ) for s in self.fragments ] +
+                 [ ( "FULL", s ) for s in self.fullStrings ] )
+
+
+    def visit_Compare(self, node):
+        opstype = str(type(node.ops[0]))
+        if opstype in ["<class '_ast.Eq'>","<class '_ast.NotEq'>"]:
+            leftClass = str(type(node.left))
+            rightClass = str(type(node.comparators[0]))
+            if leftClass == "<class '_ast.Str'>" and rightClass != "<class '_ast.Str'>":
+                self.fullStrings.add(node.left.s)
+            elif leftClass != "<class '_ast.Str'>" and rightClass == "<class '_ast.Str'>":
+                self.fullStrings.add(node.comparators[0].s)
+        elif opstype == "<class '_ast.In'>":
+            leftClass = str(type(node.left))
+            comparatorsClass = str(type(node.comparators))
+            if leftClass == "<class '_ast.Str'>":
+                self.fragments.add(node.left.s)
+            elif comparatorsClass == "<class 'list'>":
+                for element in node.comparators[0].elts:
+                    elementClass = str(type(element))
+                    if elementClass == "<class '_ast.Str'>":
+                        self.fullStrings.add(element.s)
+
+
+    def visit_Call(self,node):
+        if node.func.attr == "startswith" and str(type(node.args[0])) == "<class '_ast.Str'>":
+            self.prefixes.add(node.args[0].s)
+        elif node.func.attr == "endswith" and str(type(node.args[0])) == "<class '_ast.Str'>":
+            self.suffixes.add(node.args[0].s)
+        elif node.func.attr == "index" and str(type(node.args[0])) == "<class '_ast.Str'>":
+            self.fragments.add(node.args[0].s)
+        elif node.func.attr == "find" and str(type(node.args[0])) == "<class '_ast.Str'>":
+            self.fragments.add(node.args[0].s)

Diff for: tests/stringprocessor-testdata.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+
+# Line 5: single-line statement
+print("Hello")
+
+# Line 8: two line statement with implicit continuation
+print(
+"Hello")
+
+# Line 12: three line statement with implicit continuation
+print(
+"Hello"
+)
+
+# Line 17: two line statement with explicit continuation
+print\
+("Hello")
+
+# Line 21: three line statement with explicit continuation
+print\
+(\
+"Hello")
+
+# Line 26: if statement / line 28 elif statement / line 30 else statement
+if foo == "bar":
+    print("Hello")
+elif foo == "baz":
+    print("Bye")
+else:
+    print("Hello")
+
+# Line 34: while loop
+while foo == "bat":
+    print("Foo is bat")
+
+# Line 38: for loop
+for a in b:
+    print("Foo bar")
+
+# Line 42,44,46: try / except / finally
+try:
+    print("Foo")
+except Exception as e:
+    print("Bar")
+finally:
+    print("Baz")

Diff for: tests/testfile_jinja.j2

@@ -0,0 +1 @@
+foo