Remove sklearn from contrib tests

Fixes open-telemetry#4063

Author: ocelotl

Diff for: .github/workflows/test.yml

@@ -174,7 +174,6 @@ jobs:
           - "redis"
           - "remoulade"
           - "requests"
-          - "sklearn"
           - "sqlalchemy"
           - "sqlite3"
           - "starlette"