Merging dev to master for release 1.0.0a20 (#141)

MrMeemus · web-flow · commit ad370356667f · 2017-09-01T18:56:18.000-04:00
* Use SqlToolsService built on .NET Core 2.0 and a build script updates (#131) * Bump version to 1.0.0a19 * Use .NET Core 2.0 RTM built sqltoolsservice * Add build script to upload to azure blob storage * Upgrade to VS 2017 * Remove 3.3 as supported Python version * Fix perf issue where main event loop takes 100% of CPU (#132) Fix perf issue where main event loop takes 100% of CPU We have a 2 threads: Thread #1 runs in a loop polling the response queue Thread #2 runs in a loop decoding responses from the sqltoolsservice over stdout and posting them to the response queue Since thread #1 doesn't sleep, it's takes 100% CPU. In addition, running python 2.7 on windows, #2 doesn’t preempt the CPU due to #1 taking all of the CPU cycles, so no response is processed. Fix is simple – thread #1 needs to sleep so thread #2 can get scheduled and get it’s work done. * Refine event loop perf fix in main.py Refine event loop perf fix in main.py * Fixing regular expression Previous regex would result in release:a1 and release_version: 12. Modified the regex for part Release to only pick up lower case letters. * Adding missing forward slash on test pypi url * fixing typos/grammar (#138) fixing typos/grammar. * Updating to release version 1.0.0a20.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,6 +1,6 @@
 [bumpversion]
-current_version = 1.0.0a18
-parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>.*))(?P<release_version>\d+)
+current_version = 1.0.0a20
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)((?P<release>[a-z]+))(?P<release_version>\d+)
 serialize = 
 	{major}.{minor}.{patch}{release}{release_version}
 
diff --git a/.travis.yml b/.travis.yml
@@ -10,8 +10,6 @@ matrix:
       env: TOXENV=py27
     - os: linux
       python: "2.7"
-    - os: linux
-      python: "3.3"
     - os: linux
       python: "3.4"
     - os: linux
diff --git a/appveyor.yml b/appveyor.yml
@@ -3,8 +3,6 @@ environment:
   matrix:
     - TOXENV: "py27"
       PYTHON: "C:\\Python27"
-    - TOXENV: "py33"
-      PYTHON: "C:\\Python33"
     - TOXENV: "py34"
       PYTHON: "C:\\Python34"
     - TOXENV: "py35"
diff --git a/build.py b/build.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from __future__ import print_function
+import os
+import re
+import sys
+import tempfile
+import utility
+from azure.storage.blob import BlockBlobService, ContentSettings
+
+AZURE_STORAGE_CONNECTION_STRING = os.environ.get('AZURE_STORAGE_CONNECTION_STRING')
+BLOB_CONTAINER_NAME = 'simple'
+UPLOADED_PACKAGE_LINKS = [] 
+
+
+def print_heading(heading, f=None):
+    print('{0}\n{1}\n{0}'.format('=' * len(heading), heading), file=f)
+
+
+def upload_index_file(service, blob_name, title, links):
+    print('Uploading index file {}'.format(blob_name))
+    service.create_blob_from_text(
+        container_name=BLOB_CONTAINER_NAME,
+        blob_name=blob_name,
+        text="<html><head><title>{0}</title></head><body><h1>{0}</h1>{1}</body></html>"
+            .format(title, '\n'.join(
+                ['<a href="{0}">{0}</a><br/>'.format(link) for link in links])),
+        content_settings=ContentSettings(
+            content_type='text/html',
+            content_disposition=None,
+            content_encoding=None,
+            content_language=None,
+            content_md5=None,
+            cache_control=None
+        )
+    )
+
+
+def gen_pkg_index_html(service, pkg_name):
+    links = []
+    index_file_name = pkg_name+'/'
+    for blob in list(service.list_blobs(BLOB_CONTAINER_NAME, prefix=index_file_name)):
+        if blob.name == index_file_name:
+            # Exclude the index file from being added to the list
+            continue
+        links.append(blob.name.replace(index_file_name, ''))
+    upload_index_file(service, index_file_name, 'Links for {}'.format(pkg_name), links)
+    UPLOADED_PACKAGE_LINKS.append(index_file_name)
+
+
+def upload_package(service, file_path, pkg_name):
+    print('Uploading {}'.format(file_path))
+    file_name = os.path.basename(file_path)
+    blob_name = '{}/{}'.format(pkg_name, file_name)
+    service.create_blob_from_path(
+        container_name=BLOB_CONTAINER_NAME,
+        blob_name=blob_name,
+        file_path=file_path
+    )
+    gen_pkg_index_html(service, pkg_name)
+
+
+def build(options):
+
+    supported_actions = ['nightly']
+    action = None
+
+    if len(options) >= 1:
+        if options[0] not in supported_actions:
+            print('Please provide a supported action {}.'.format(supported_actions))
+            return
+        action = options[0]
+
+    if action == 'nightly':
+        assert AZURE_STORAGE_CONNECTION_STRING, 'Set AZURE_STORAGE_CONNECTION_STRING environment variable'
+
+    print_heading('Cleanup')
+
+    # clean
+    utility.clean_up(utility.MSSQLSCRIPTER_DIST_DIRECTORY)
+    utility.clean_up(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY)
+    utility.cleaun_up_egg_info_sub_directories(utility.ROOT_DIR)
+    utility.cleaun_up_egg_info_sub_directories(utility.MSSQLTOOLSSERVICE_DIRECTORY)
+    
+    print_heading('Running setup')
+    
+    # install general requirements.
+    utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR)
+    
+    print_heading('Running mssql-scripter tests')
+    utility.exec_command('tox', utility.ROOT_DIR, continue_on_error = False)
+    
+    print_heading('Building mssql-scripter pip package')
+    utility.exec_command('python setup.py check -r -s sdist', utility.ROOT_DIR, continue_on_error = False)
+    
+    print_heading('Building mssqltoolsservice pip package')
+    utility.exec_command('python buildwheels.py', utility.MSSQLTOOLSSERVICE_DIRECTORY, continue_on_error = False)
+
+    if action == 'nightly':
+        blob_service = BlockBlobService(connection_string=AZURE_STORAGE_CONNECTION_STRING)
+    
+        print_heading('Uploading packages to blob storage ')
+        for pkg in os.listdir(utility.MSSQLSCRIPTER_DIST_DIRECTORY):
+            pkg_path = os.path.join(utility.MSSQLSCRIPTER_DIST_DIRECTORY, pkg)
+            print('Uploading package {}'.format(pkg_path))
+            upload_package(blob_service, pkg_path, 'mssql-scripter')
+    
+        for pkg in os.listdir(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY):
+            pkg_path = os.path.join(utility.MSSQLTOOLSSERVICE_DIST_DIRECTORY, pkg)
+            pkg_name = os.path.basename(pkg_path).split('-')[0].replace('_', '-').lower()
+            print('Uploading package {}'.format(pkg_name))
+            upload_package(blob_service, pkg_path, pkg_name)
+
+        # Upload the final index file
+        upload_index_file(blob_service, 'index.html', 'Simple Index', UPLOADED_PACKAGE_LINKS)
+
+
+if __name__ == '__main__':
+    build(sys.argv[1:])
diff --git a/dev_requirements.txt b/dev_requirements.txt
@@ -11,4 +11,5 @@ flake8 >= 3.3.0
 pytest >= 3.0.7
 pytest-cov >= 2.5.1
 readme_renderer >= 17.2
-docutils >= 0.13.1
+docutils >= 0.13.1
+azure-storage >= 0.33.0
diff --git a/dev_setup.py b/dev_setup.py
@@ -12,18 +12,16 @@
 import setup
 import utility
 
-root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))
-
 print('Running dev setup...')
-print('Root directory \'{}\'\n'.format(root_dir))
+print('Root directory \'{}\'\n'.format(utility.ROOT_DIR))
 
 # install general requirements.
-utility.exec_command('pip install -r dev_requirements.txt', root_dir)
+utility.exec_command('pip install -r dev_requirements.txt', utility.ROOT_DIR)
 
 # install mssqltoolsservice if this platform supports it.
 mssqltoolsservice_package_name = os.environ['MSSQLTOOLSSERVICE_PACKAGE_NAME']
 print('Installing {}...'.format(mssqltoolsservice_package_name))
 # mssqltoolsservice package name is retrieved from environment variable set by setup.py.
-utility.exec_command('pip install {}'.format(mssqltoolsservice_package_name), root_dir)
+utility.exec_command('pip install {}'.format(mssqltoolsservice_package_name), utility.ROOT_DIR)
 
 print('Finished dev setup.')
diff --git a/doc/installation_guide.md b/doc/installation_guide.md
@@ -1,7 +1,7 @@
 # Installation Guide
 
 ## Quick Start
-mssql-scritper is installed via pip.  If you know  pip, you can install mssql-scripter using command
+mssql-scripter is installed via pip.  If you know pip, you can install mssql-scripter using command
 ```shell
 $ pip install mssql-scripter 
 ```
@@ -70,7 +70,7 @@ $ sudo apt-get install python-pip
 $ sudo pip install --upgrade pip
 ```
 
-Install mssql-scritper using command:
+Install mssql-scripter using command:
 
 ```shell
 $ sudo pip install mssql-scripter 
@@ -103,7 +103,7 @@ More information can be found at:
 
 - [Development guide](development_guide.md#Environment_Setup)
 
-## Error: Could not find version that satifies the requirement mssql-scripter
+## Error: Could not find version that satisfies the requirement mssql-scripter
 If you see the above error running `pip install mssql-scripter`, this means the pip version used is out-of-date.  Upgrade pip using the command:
 ```shell
 $ sudo apt-get install python-pip
@@ -141,7 +141,7 @@ $ sudo apt-get install libunwind8
 ```
 
 ### Debian 8
-The file `/etc/apt/sources.list' needs to updated with the following line
+The file `/etc/apt/sources.list' needs to be updated with the following line
 ```
 deb http://ftp.us.debian.org/debian/ jessie main
 ```
diff --git a/doc/pypi_release_steps.md b/doc/pypi_release_steps.md
@@ -86,7 +86,7 @@ bumpversion release_version    ->  1.0.0a<b>1</b>
 		    pypitest
 		 
 		[pypitest]
-		repository = https://test.pypi.org/legacy
+		repository = https://test.pypi.org/legacy/
 		username = your_username
 		password = your_password
         ```
diff --git a/mssqlscripter/__init__.py b/mssqlscripter/__init__.py
@@ -3,4 +3,4 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-__version__ = '1.0.0a18'
+__version__ = '1.0.0a20'
diff --git a/mssqlscripter/jsonrpc/contracts/tests/test_scripting.py b/mssqlscripter/jsonrpc/contracts/tests/test_scripting.py
@@ -362,7 +362,7 @@ def generate_new_baseline(self, file_name):
             # Point sqltoolsservice output to file.
             with io.open(file_name, 'wb') as baseline:
                 tools_service_process = subprocess.Popen(
-                    'D:\\GitHub\\sqltoolsservice\\src\\Microsoft.SqlTools.ServiceLayer\\bin\\Debug\\netcoreapp1.0\\win7-x64\\Microsoft.SqlTools.ServiceLayer.exe',
+                    'D:\\GitHub\\sqltoolsservice\\src\\Microsoft.SqlTools.ServiceLayer\\bin\\Debug\\netcoreapp2.0\\win7-x64\\MicrosoftSqlToolsServiceLayer.exe',
                     bufsize=0,
                     stdin=subprocess.PIPE,
                     stdout=baseline)
diff --git a/mssqlscripter/main.py b/mssqlscripter/main.py
@@ -87,9 +87,11 @@ def main(args):
 
         while not scripting_request.completed():
             response = scripting_request.get_response()
-
             if response:
                 scriptercallbacks.handle_response(response, parameters.DisplayProgress)
+            else:
+                # The sleep prevents burning up the CPU and lets other threads get scheduled.
+                time.sleep(0.1)
 
         # Only write to stdout if user did not provide a file path.
         logger.info('stdout current encoding: {}'.format(sys.stdout.encoding))
diff --git a/mssqltoolsservice/buildwheels.py b/mssqltoolsservice/buildwheels.py
@@ -17,20 +17,20 @@
 from urllib.request import urlopen
 
 
-DOWNLOAD_URL_BASE = 'https://mssqlscripter.blob.core.windows.net/sqltoolsservice-08-01-2017/'
+DOWNLOAD_URL_BASE = 'https://mssqlscripter.blob.core.windows.net/sqltoolsservice-08-16-2017/'
 
 # Supported platform key's must match those in mssqlscript's setup.py.
 SUPPORTED_PLATFORMS = {
-    'CentOS_7': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-centos-x64-netcoreapp1.0.tar.gz',
-    'Debian_8': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-debian-x64-netcoreapp1.0.tar.gz',
-    'Fedora_23': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-fedora-x64-netcoreapp1.0.tar.gz',
-    'openSUSE_13_2': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-opensuse-x64-netcoreapp1.0.tar.gz',
-    'OSX_10_11_64': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-osx-x64-netcoreapp1.0.tar.gz',
-    'RHEL_7': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-rhel-x64-netcoreapp1.0.tar.gz',
-    'Ubuntu_14': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-ubuntu14-x64-netcoreapp1.0.tar.gz',
-    'Ubuntu_16': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-ubuntu16-x64-netcoreapp1.0.tar.gz',
-    'Windows_7_64': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-win-x64-netcoreapp1.0.zip',
-    'Windows_7_86': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-win-x86-netcoreapp1.0.zip',
+    'CentOS_7': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-centos-x64-netcoreapp2.0.tar.gz',
+    'Debian_8': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-debian-x64-netcoreapp2.0.tar.gz',
+    'Fedora_23': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-fedora-x64-netcoreapp2.0.tar.gz',
+    'openSUSE_13_2': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-opensuse-x64-netcoreapp2.0.tar.gz',
+    'OSX_10_11_64': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-osx-x64-netcoreapp2.0.tar.gz',
+    'RHEL_7': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-rhel-x64-netcoreapp2.0.tar.gz',
+    'Ubuntu_14': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-ubuntu14-x64-netcoreapp2.0.tar.gz',
+    'Ubuntu_16': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-ubuntu16-x64-netcoreapp2.0.tar.gz',
+    'Windows_7_64': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-win-x64-netcoreapp2.0.zip',
+    'Windows_7_86': DOWNLOAD_URL_BASE + 'microsoft.sqltools.servicelayer-win-x86-netcoreapp2.0.zip'
 }
 
 CURRENT_DIRECTORY = os.path.abspath(os.path.join(os.path.abspath(__file__), '..'))
diff --git a/mssqltoolsservice/mssqltoolsservice/__init__.py b/mssqltoolsservice/mssqltoolsservice/__init__.py
@@ -9,7 +9,7 @@
 import os
 import platform
 
-__version__ = '1.0.0a18'
+__version__ = '1.0.0a20'
 
 
 def get_executable_path():
@@ -28,7 +28,7 @@ def get_executable_path():
                 'bin'))
 
     # Format name based on platform.
-    mssqltoolsservice_name = u'Microsoft.SqlTools.ServiceLayer{}'.format(
+    mssqltoolsservice_name = u'MicrosoftSqlToolsServiceLayer{}'.format(
         u'.exe' if (platform.system() == u'Windows') else u'')
 
     mssqltoolsservice_full_path = os.path.abspath(os.path.join(mssqltoolsservice_base_path, mssqltoolsservice_name))
diff --git a/mssqltoolsservice/setup.py b/mssqltoolsservice/setup.py
@@ -12,7 +12,7 @@
 
 # This version number is in place in two places and must be in sync with
 # mssqlscripter's version in setup.py.
-MSSQLTOOLSSERVICE_VERSION = '1.0.0a18'
+MSSQLTOOLSSERVICE_VERSION = '1.0.0a20'
 
 # If we have source, validate version numbers match to prevent
 # uploading releases with mismatched versions.
diff --git a/setup.py b/setup.py
@@ -14,7 +14,7 @@
 
 # This version number is in place in two places and must be in sync with
 # mssqltoolsservice's version in setup.py.
-MSSQLSCRIPTER_VERSION = '1.0.0a18'
+MSSQLSCRIPTER_VERSION = '1.0.0a20'
 
 # If we have the source, validate our setup version matches source version.
 # This will prevent uploading releases with mismatched versions. This will
@@ -56,7 +56,7 @@
                 toolsservice_version.group(1)))
         sys.exit(1)
 
-MSSQLTOOLSSERVICE_PACKAGE_NAME = 'mssqltoolsservice_{}=={}'
+MSSQLTOOLSSERVICE_PACKAGE_NAME = 'mssqltoolsservice-{}=={}'
 MSSQLTOOLSSERVICE_PACKAGE_SUFFIX = [
     'CentOS_7',
     'Debian_8',
@@ -204,7 +204,7 @@ def get_mssqltoolsservice_package_name(run_time_id=_get_runtime_id()):
         # set package suffix name for other uses like building wheels outside of setup.py.
         os.environ['MSSQLTOOLSSERVICE_PACKAGE_SUFFIX'] = run_time_id
         return MSSQLTOOLSSERVICE_PACKAGE_NAME.format(
-            run_time_id, MSSQLSCRIPTER_VERSION)
+            run_time_id, MSSQLSCRIPTER_VERSION).replace('_', '-').lower()
 
     raise EnvironmentError('mssqltoolsservice is not supported on this platform.')
 
@@ -217,7 +217,6 @@ def get_mssqltoolsservice_package_name(run_time_id=_get_runtime_id()):
     'Programming Language :: Python :: 2',
     'Programming Language :: Python :: 2.7',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.3',
     'Programming Language :: Python :: 3.4',
     'Programming Language :: Python :: 3.5',
     'Programming Language :: Python :: 3.6',
diff --git a/sql-xplat-cli.pyproj b/sql-xplat-cli.pyproj
@@ -11,8 +11,6 @@
     <OutputPath>.</OutputPath>
     <ProjectTypeGuids>{888888a0-9f3d-457c-b088-3a5042f75d52}</ProjectTypeGuids>
     <LaunchProvider>Standard Python launcher</LaunchProvider>
-    <InterpreterId />
-    <InterpreterVersion />
     <CommandLineArguments>-S localhost -d AdventureWorks2014</CommandLineArguments>
     <EnableNativeCodeDebugging>False</EnableNativeCodeDebugging>
     <IsWindowsApplication>False</IsWindowsApplication>
@@ -21,10 +19,11 @@
   <PropertyGroup Condition="'$(Configuration)' == 'Release'" />
   <PropertyGroup>
     <VisualStudioVersion Condition=" '$(VisualStudioVersion)' == '' ">10.0</VisualStudioVersion>
-    <PtvsTargetsFile>$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets</PtvsTargetsFile>
   </PropertyGroup>
   <ItemGroup>
     <Content Include=".gitignore" />
+    <Content Include="appveyor.yml" />
+    <Content Include="dev_requirements.txt" />
     <Content Include="doc\README.md" />
     <Content Include="doc\architecture_guide.md" />
     <Content Include="doc\development_guide.md" />
@@ -43,13 +42,13 @@
     <Content Include="mssqltoolsservice\README.rst" />
     <Content Include="mssqltoolsservice\setup.cfg" />
     <Content Include="README.rst" />
-    <Content Include="requirements.txt" />
     <Content Include="setup.cfg" />
     <Content Include="tox.ini" />
     <Content Include=".bumpversion.cfg" />
     <Content Include=".travis.yml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="build.py" />
     <Compile Include="dev_setup.py" />
     <Compile Include="mssqlscripter\argparser.py" />
     <Compile Include="mssqlscripter\jsonrpc\contracts\scriptingservice.py" />
@@ -87,6 +86,5 @@
     <Folder Include="mssqltoolsservice\" />
     <Folder Include="mssqltoolsservice\mssqltoolsservice\" />
   </ItemGroup>
-  <Import Project="$(PtvsTargetsFile)" Condition="Exists($(PtvsTargetsFile))" />
-  <Import Project="$(MSBuildToolsPath)\Microsoft.Common.targets" Condition="!Exists($(PtvsTargetsFile))" />
+  <Import Project="$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)\Python Tools\Microsoft.PythonTools.targets" />
 </Project>
diff --git a/sql-xplat-cli.sln b/sql-xplat-cli.sln
diff --git a/utility.py b/utility.py
