Merge pull request #16252 from RasmusWL/move-dataflow-tests

Python: Move dataflow tests out of experimental

Author: RasmusWL

config/identical-files.json

@@ -362,7 +362,7 @@
     "java/ql/lib/semmle/code/java/security/internal/EncryptionKeySizes.qll"
   ],
   "Python model summaries test extension": [
-    "python/ql/test/experimental/dataflow/model-summaries/InlineTaintTest.ext.yml",
-    "python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ext.yml"
+    "python/ql/test/library-tests/dataflow/model-summaries/InlineTaintTest.ext.yml",
+    "python/ql/test/library-tests/dataflow/model-summaries/NormalDataflowTest.ext.yml"
   ]
-}
+}

python/ql/lib/semmle/python/dataflow/new/SensitiveDataSources.qll

@@ -89,7 +89,7 @@ private module SensitiveDataModeling {
    */
   DataFlow::Node sensitiveLookupStringConst(SensitiveDataClassification classification) {
     // Note: If this is implemented with type-tracking, we will get cross-talk as
-    // illustrated in python/ql/test/experimental/dataflow/sensitive-data/test.py
+    // illustrated in python/ql/test/library-tests/dataflow/sensitive-data/test.py
     exists(DataFlow::LocalSourceNode source |
       source.asExpr().(StringLiteral).getText() = sensitiveString(classification) and
       source.flowsTo(result)

python/ql/lib/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -638,7 +638,7 @@ newtype TContent =
     //   name = any(AccessPathToken a).getAnArgument("Attribute")
     // instead we use a qltest to alert if we write a new summary in QL that uses an
     // attribute -- see
-    // python/ql/test/experimental/dataflow/summaries-checks/missing-attribute-content.ql
+    // python/ql/test/library-tests/dataflow/summaries-checks/missing-attribute-content.ql
     attr in ["re", "string", "pattern"]
     or
     //

python/ql/test/experimental/dataflow/TestUtil/DataflowQueryTest.qll renamed to python/ql/test/TestUtilities/dataflow/DataflowQueryTest.qll

@@ -1,6 +1,6 @@
 import python
-import experimental.dataflow.TestUtil.FlowTest
-import experimental.dataflow.testConfig
+import TestUtilities.dataflow.FlowTest
+import TestUtilities.dataflow.testConfig
 private import semmle.python.dataflow.new.internal.PrintNode
 
 module DataFlowTest implements FlowTestSig {

python/ql/test/experimental/dataflow/TestUtil/FlowTest.qll renamed to python/ql/test/TestUtilities/dataflow/FlowTest.qll

@@ -1,6 +1,6 @@
 import python
-import experimental.dataflow.TestUtil.FlowTest
-import experimental.dataflow.testTaintConfig
+import TestUtilities.dataflow.FlowTest
+import TestUtilities.dataflow.testTaintConfig
 private import semmle.python.dataflow.new.internal.PrintNode
 
 module DataFlowTest implements FlowTestSig {

python/ql/test/experimental/dataflow/TestUtil/LocalFlowStepTest.qll renamed to python/ql/test/TestUtilities/dataflow/LocalFlowStepTest.qll

@@ -9,7 +9,7 @@
 // 3. if necessary, look at partial paths by (un)commenting appropriate lines
 import python
 import semmle.python.dataflow.new.DataFlow
-import experimental.dataflow.testConfig
+import TestUtilities.dataflow.testConfig
 
 module Config implements DataFlow::ConfigSig {
   predicate isSource(DataFlow::Node source) { TestConfig::isSource(source) }

python/ql/test/experimental/dataflow/TestUtil/MaximalFlowTest.qll renamed to python/ql/test/TestUtilities/dataflow/MaximalFlowTest.qll

@@ -1,4 +1,4 @@
 import python
-import experimental.dataflow.TestUtil.DataflowQueryTest
+import TestUtilities.dataflow.DataflowQueryTest
 import experimental.Security.UnsafeUnpackQuery
 import FromTaintTrackingConfig<UnsafeUnpackConfig>

python/ql/test/experimental/dataflow/TestUtil/NormalDataflowTest.qll renamed to python/ql/test/TestUtilities/dataflow/NormalDataflowTest.qll

@@ -1,4 +1,4 @@
 import python
-import experimental.dataflow.TestUtil.DataflowQueryTest
+import TestUtilities.dataflow.DataflowQueryTest
 import experimental.semmle.python.security.DecompressionBomb
 import FromTaintTrackingConfig<BombsConfig>

python/ql/test/experimental/dataflow/TestUtil/NormalTaintTrackingTest.qll renamed to python/ql/test/TestUtilities/dataflow/NormalTaintTrackingTest.qll

@@ -1,4 +1,4 @@
-import experimental.dataflow.callGraphConfig
+import TestUtilities.dataflow.callGraphConfig
 
 from DataFlow::Node source, DataFlow::Node sink
 where

python/ql/test/experimental/dataflow/TestUtil/RoutingTest.qll renamed to python/ql/test/TestUtilities/dataflow/RoutingTest.qll

@@ -1,4 +1,4 @@
-import experimental.dataflow.callGraphConfig
+import TestUtilities.dataflow.callGraphConfig
 
 from DataFlow::Node sink
 where

python/ql/test/experimental/dataflow/TestUtil/UnresolvedCalls.qll renamed to python/ql/test/TestUtilities/dataflow/UnresolvedCalls.qll

@@ -1,4 +1,4 @@
-import experimental.dataflow.callGraphConfig
+import TestUtilities.dataflow.callGraphConfig
 
 from DataFlow::Node source
 where

python/ql/test/experimental/dataflow/callGraphConfig.qll renamed to python/ql/test/TestUtilities/dataflow/callGraphConfig.qll

@@ -0,0 +1 @@
+import TestUtilities.dataflow.LocalFlowStepTest

python/ql/test/experimental/dataflow/testConfig.qll renamed to python/ql/test/TestUtilities/dataflow/testConfig.qll

@@ -0,0 +1 @@
+import TestUtilities.dataflow.MaximalFlowTest

python/ql/test/experimental/dataflow/testTaintConfig.qll renamed to python/ql/test/TestUtilities/dataflow/testTaintConfig.qll

@@ -0,0 +1,2 @@
+import python
+import TestUtilities.dataflow.NormalDataflowTest

python/ql/test/experimental/dataflow/basic/localFlowStepTest.ql

@@ -1,7 +1,7 @@
 import python
 import semmle.python.dataflow.new.DataFlow
 private import semmle.python.dataflow.new.internal.DataFlowPrivate as DataFlowPrivate
-import experimental.dataflow.TestUtil.RoutingTest
+import TestUtilities.dataflow.RoutingTest
 
 module Argument1RoutingTest implements RoutingTestSig {
   class Argument = Unit;

python/ql/test/experimental/dataflow/basic/maximalFlowTest.ql

@@ -6,7 +6,7 @@
 # A thorough covering of methods in that document is found in classes.py.
 #
 # Intended sources should be the variable `SOURCE` and intended sinks should be
-# arguments to the function `SINK` (see python/ql/test/experimental/dataflow/testConfig.qll).
+# arguments to the function `SINK` (see python/ql/test/library-tests/dataflow/testConfig.qll).
 
 import sys
 import os

python/ql/test/experimental/dataflow/coverage/NormalDataflowTest.ql

@@ -2,7 +2,7 @@
 # Headings refer to https://docs.python.org/3/reference/expressions.html,
 # and are selected whenever they incur dataflow.
 # Intended sources should be the variable `SOURCE` and intended sinks should be
-# arguments to the function `SINK` (see python/ql/test/experimental/dataflow/testConfig.qll).
+# arguments to the function `SINK` (see python/ql/test/library-tests/dataflow/testConfig.qll).
 #
 # Functions whose name ends with "_with_local_flow" will also be tested for local flow.
 #

python/ql/test/experimental/dataflow/exceptions/NormalDataflowTest.ql

@@ -0,0 +1,2 @@
+import python
+import TestUtilities.dataflow.NormalDataflowTest

python/ql/test/experimental/dataflow/fieldflow/NormalDataflowTest.ql

@@ -0,0 +1,2 @@
+import python
+import TestUtilities.dataflow.NormalDataflowTest

python/ql/test/experimental/dataflow/match/NormalDataflowTest.ql

@@ -1,5 +1,5 @@
 import python
-import experimental.dataflow.TestUtil.UnresolvedCalls
+import TestUtilities.dataflow.UnresolvedCalls
 private import semmle.python.dataflow.new.DataFlow
 
 module IgnoreDictMethod implements UnresolvedCallExpectationsSig {

python/ql/test/experimental/dataflow/model-summaries/NormalDataflowTest.ql

@@ -0,0 +1,2 @@
+import python
+import TestUtilities.dataflow.NormalDataflowTest

python/ql/test/experimental/dataflow/summaries/NormalTaintTrackingTest.ql

@@ -0,0 +1,2 @@
+import python
+import TestUtilities.dataflow.NormalDataflowTest

python/ql/test/experimental/dataflow/tainttracking/generator-flow/NormalDataflowTest.ql

@@ -1,6 +1,6 @@
 // This query should be more focused yet.
 import python
-import experimental.dataflow.TestUtil.FlowTest
+import TestUtilities.dataflow.FlowTest
 private import semmle.python.dataflow.new.internal.PrintNode
 private import semmle.python.dataflow.new.internal.DataFlowPrivate as DP
 

python/ql/test/experimental/meta/debug/dataflowTestPaths.ql

@@ -5,7 +5,7 @@
 import python
 import semmle.python.dataflow.new.DataFlow
 import semmle.python.dataflow.new.TaintTracking
-import experimental.dataflow.testConfig
+import TestUtilities.dataflow.testConfig
 import TestUtilities.InlineExpectationsTest
 
 module TestTaintFlow = TaintTracking::Global<TestConfig>;

python/ql/test/experimental/query-tests/Security/CWE-022-UnsafeUnpacking/DataflowQueryTest.ql

@@ -6,7 +6,7 @@
  */
 
 import python
-import experimental.dataflow.testConfig
+import TestUtilities.dataflow.testConfig
 
 from DataFlow::Node source, DataFlow::Node sink
 where TestFlow::flow(source, sink)

python/ql/test/experimental/query-tests/Security/CWE-409/DataflowQueryTest.ql

@@ -0,0 +1,3 @@
+import python
+private import TestSummaries
+import TestUtilities.dataflow.NormalTaintTrackingTest

python/ql/test/experimental/dataflow/basic/allFlowsConfig.qll renamed to python/ql/test/library-tests/dataflow/basic/allFlowsConfig.qll

@@ -8,7 +8,7 @@ import TestFlow::PathGraph
 import semmle.python.dataflow.new.TaintTracking
 import semmle.python.dataflow.new.internal.FlowSummaryImpl
 import semmle.python.ApiGraphs
-import experimental.dataflow.testTaintConfig
+import TestUtilities.dataflow.testTaintConfig
 private import TestSummaries
 
 query predicate invalidSpecComponent(SummarizedCallable sc, string s, string c) {

python/ql/test/experimental/dataflow/basic/callGraph.expected renamed to python/ql/test/library-tests/dataflow/basic/callGraph.expected

@@ -0,0 +1,2 @@
+import python
+import TestUtilities.dataflow.NormalDataflowTest

python/ql/test/experimental/dataflow/basic/callGraph.ql renamed to python/ql/test/library-tests/dataflow/basic/callGraph.ql

@@ -1,7 +1,7 @@
 import python
 import semmle.python.dataflow.new.DataFlow
 import TestUtilities.InlineExpectationsTest
-import experimental.dataflow.testConfig
+import TestUtilities.dataflow.testConfig
 
 module CaptureTest implements TestSig {
   string getARelevantTag() { result = "captured" }

python/ql/test/experimental/dataflow/basic/callGraphSinks.expected renamed to python/ql/test/library-tests/dataflow/basic/callGraphSinks.expected

@@ -1,2 +1,2 @@
 import python
-import experimental.dataflow.TestUtil.NormalDataflowTest
+import TestUtilities.dataflow.NormalDataflowTest

python/ql/test/experimental/dataflow/basic/callGraphSinks.ql renamed to python/ql/test/library-tests/dataflow/basic/callGraphSinks.ql

@@ -1,6 +1,6 @@
 import base64
 
-# TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py
+# TODO: These tests should be merged with python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py
 base64.a85decode(payload)  # $ decodeInput=payload decodeOutput=base64.a85decode(..) decodeFormat=Ascii85
 base64.b85decode(payload)  # $ decodeInput=payload decodeOutput=base64.b85decode(..) decodeFormat=Base85
 base64.decodebytes(payload)  # $ decodeInput=payload decodeOutput=base64.decodebytes(..) decodeFormat=Base64

python/ql/test/experimental/dataflow/basic/callGraphSources.expected renamed to python/ql/test/library-tests/dataflow/basic/callGraphSources.expected

@@ -1,6 +1,6 @@
 import base64
 
-# TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py
+# TODO: These tests should be merged with python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep-py3/test_string.py
 base64.a85encode(bs) # $ encodeInput=bs encodeOutput=base64.a85encode(..) encodeFormat=Ascii85
 base64.b85encode(bs)# $ encodeInput=bs encodeOutput=base64.b85encode(..) encodeFormat=Base85
 base64.encodebytes(bs)# $ encodeInput=bs encodeOutput=base64.encodebytes(..) encodeFormat=Base64

python/ql/test/experimental/dataflow/basic/callGraphSources.ql renamed to python/ql/test/library-tests/dataflow/basic/callGraphSources.ql

@@ -26,7 +26,7 @@
 shelve.open(filepath)  # $ decodeInput=filepath decodeOutput=shelve.open(..) decodeFormat=pickle decodeMayExecuteInput getAPathArgument=filepath
 shelve.open(filename=filepath)  # $ decodeInput=filepath decodeOutput=shelve.open(..) decodeFormat=pickle decodeMayExecuteInput getAPathArgument=filepath
 
-# TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py
+# TODO: These tests should be merged with python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py
 base64.b64decode(payload)  # $ decodeInput=payload decodeOutput=base64.b64decode(..) decodeFormat=Base64
 base64.standard_b64decode(payload)  # $ decodeInput=payload decodeOutput=base64.standard_b64decode(..) decodeFormat=Base64
 base64.urlsafe_b64decode(payload)  # $ decodeInput=payload decodeOutput=base64.urlsafe_b64decode(..) decodeFormat=Base64

python/ql/test/experimental/dataflow/basic/global.expected renamed to python/ql/test/library-tests/dataflow/basic/global.expected

@@ -5,7 +5,7 @@
 pickle.dumps(obj)  # $ MISSING: encodeInput=obj encodeOutput=pickle.dumps(..) encodeFormat=pickle encodeMayExecuteInput
 marshal.dumps(obj)  # $ MISSING: encodeInput=obj encodeOutput=marshal.dumps(..) encodeFormat=marshal encodeMayExecuteInput
 
-# TODO: These tests should be merged with python/ql/test/experimental/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py
+# TODO: These tests should be merged with python/ql/test/library-tests/dataflow/tainttracking/defaultAdditionalTaintStep/test_string.py
 base64.b64encode(bs)  # $ encodeInput=bs encodeOutput=base64.b64encode(..) encodeFormat=Base64
 base64.standard_b64encode(bs)  # $ encodeInput=bs encodeOutput=base64.standard_b64encode(..) encodeFormat=Base64
 base64.urlsafe_b64encode(bs)  # $ encodeInput=bs encodeOutput=base64.urlsafe_b64encode(..) encodeFormat=Base64

python/ql/test/experimental/dataflow/basic/global.ql renamed to python/ql/test/library-tests/dataflow/basic/global.ql

@@ -1,4 +1,4 @@
 import python
-import experimental.dataflow.TestUtil.DataflowQueryTest
+import TestUtilities.dataflow.DataflowQueryTest
 import semmle.python.security.dataflow.PathInjectionQuery
 import FromTaintTrackingStateConfig<PathInjectionConfig>

python/ql/test/experimental/dataflow/basic/globalStep.expected renamed to python/ql/test/library-tests/dataflow/basic/globalStep.expected

@@ -1,4 +1,4 @@
 import python
-import experimental.dataflow.TestUtil.DataflowQueryTest
+import TestUtilities.dataflow.DataflowQueryTest
 import semmle.python.security.dataflow.CommandInjectionQuery
 import FromTaintTrackingConfig<CommandInjectionConfig>

python/ql/test/experimental/dataflow/basic/globalStep.ql renamed to python/ql/test/library-tests/dataflow/basic/globalStep.ql

@@ -1,4 +1,4 @@
 import python
-import experimental.dataflow.TestUtil.DataflowQueryTest
+import TestUtilities.dataflow.DataflowQueryTest
 import semmle.python.security.dataflow.UnsafeShellCommandConstructionQuery
 import FromTaintTrackingConfig<UnsafeShellCommandConstructionConfig>

python/ql/test/experimental/dataflow/basic/local.expected renamed to python/ql/test/library-tests/dataflow/basic/local.expected

@@ -1,4 +1,4 @@
 import python
-import experimental.dataflow.TestUtil.DataflowQueryTest
+import TestUtilities.dataflow.DataflowQueryTest
 import semmle.python.security.dataflow.NoSqlInjectionQuery
 import FromTaintTrackingStateConfig<NoSqlInjectionConfig>