diff --git a/Nuget/CHANGELOG.md b/Nuget/CHANGELOG.md index 1b5ba99d..7c728d75 100644 --- a/Nuget/CHANGELOG.md +++ b/Nuget/CHANGELOG.md @@ -1,8 +1,16 @@ - # Introduction This file will log substantial changes made to SMO between public releases to nuget.org. +## 161.47027.0 + +- Fix distribution columns on scripting for taking into consideration more than one distribution column +- Add new EXTGOV_OPERATION_GROUP audit action type +- Force [quoted identifier on](https://github.com/microsoft/sqlmanagementobjects/issues/96) for table scripts +- Fix EnumObjects sort by Schema +- Add new enumeration values for SQL Server 2022 permissions +- Script FIRST_ROW support for serverless Synapse + ## 161.47021.0 - Add `LedgerViewSchema` property to table objects @@ -24,10 +32,10 @@ This file will log substantial changes made to SMO between public releases to nu - Add `LoginType` property to `ILoginOptions` interface. - `Login.PasswordPolicyEnforced` now returns `false` for Windows logins instead of throwing an exception - Remove net461 binaries from nuget packages -- Added Scripting Support for Ledger tables for SQL 2022+ +- Added Scripting Support for Ledger tables for Sql 2022+ - Change the `Size` property on `Server/Drive` objects to `System.Int64`. These objects don't have a C# wrapper class so it's not breaking any compilation. -- Add support for SQL Server version 16 -- Add new permissions for SQL 2019+ to SMO enumerations +- Add support for Sql Server version 16 +- Add new permissions for Sql 2019+ to SMO enumerations - Added External Stream object and External Streaming Jobs object for scripting - Add support for XML compression @@ -43,7 +51,7 @@ This file will log substantial changes made to SMO between public releases to nu ## 161.46437.65 - Update Microsoft.Data.SqlClient dependency to version 3.0.0 -- Added Scripting Support for Ledger table in Azure SQLDB +- Added Scripting Support for Ledger table in Azure Sql db - Change `Server.MasterDBPath` and `Server.MasterDBLogPath` properties to use `file_id` instead of `name` from `sys.database_files` - Enable Index creation for memory optimized tables in Azure - Fix Server/Logins to show external Logins for Azure SQLDB as they are now supported @@ -122,7 +130,65 @@ This file will log substantial changes made to SMO between public releases to nu - Enabled Server.EnumServerAttributes API on Azure SQL Database - Enabled Lock enumeration APIs on Azure SQL Database - Deleted the Database.CheckIdentityValues API -- Added new property "RequestMaximumMemoryGrantPercentageAsDouble" in WorkloadGroup to accept decimal values in Resource Governor (SQL 2019+). -- Changed the netfx binaries in Microsoft.SqlServer.SqlManagementObjects package to use Microsoft.Data.SqlClient -- Added a new package, Microsoft.SqlServer.SqlManagementObjects.SSMS, which only has netfx binaries and that uses System.Data.SqlClient - +- Added new property "RequestMaximumMemoryGrantPercentageAsDouble" in WorkloadGroup to accept decimal values in Resource Governor (SQL2019 and above). +- Fixed a scripting issue with statistics on filtered indexes where the filter from the index would be scripted with the UPDATE STATISTICS TSQL. +- Enabled Security Policy and Security Predicate objects on Azure SQL DataWarehouse + +## 160.2004021.0 + +- First non-preview 160 release, aligned with [SQL Server Management Studio](https://aka.ms/ssmsfullsetup) 18.5 +- Script extended properties for Azure SQL Database objects +- Enable Jupyter Notebook output for SqlScriptPublishModel. SSMS 18.5 can output a Notebook for Azure Data Studio in Generate Scripts now. +- Fix issue where Table.EnableAllIndexes(Recreate) did nothing +- Fix Database.EnumObjectPermissions usage in NetStandard binaries +- Remove FORCE ORDER hint from table enumeration that was causing major performance issues +- Fix Transfer with PrefetchAllObjects == false for pre-Sql 2014 versions so it doesn't throw an exception +- Extend value range for platform, name, and engineEdition JSON properties of SQL Assessment targets with arrays of strings: + + ```JSON + "target": { + "platform": ["Windows", "Linux"], + "name": ["master", "temp"] + } + ``` + +- Add 13 new [SQL Assessment rules](https://github.com/microsoft/sql-server-samples/blob/master/samples/manage/sql-assessment-api/release-notes.md) +- Fix help link in XTPHashAvgChainBuckets SQL Assessment rule +- Units for threshold parameter of FullBackup SQL Assessment rule changed from hours to days + +## 160.201141.0-preview + +- Remove unneeded "using" TSQL statements from Database.CheckTables method implementations +- Enable ColumnMasterKey properties Signature and AllowEnclaveComputations for Azure SQL DB +- Fix Database.EncryptionEnabled and Database.DatabaseEncryptionKey behavior during Database.Alter(). Now, this code will correctly create a new key using the server certificate named MyCertificate: + + ```C# + db.EncryptionEnabled = true; + db.DatabaseEncryptionKey.EncryptorName = "MyCertificate"; + db.DatabaseEncryptionKey.EncryptionAlgorithm = DatabaseEncryptionAlgorithm.Aes256; + db.DatabaseEncryptionKey.EncryptionType = DatabaseEncryptionType.ServerCertificate; + db.Alter() + ``` + +- Fixed the "like" and "contains" URN filter functions to work with parameters containing single quotes. These operators can be used to optimally initialize collections: + + ```C# + // populate the collection with databases that have Name starting with "RDA" + var server = Server(new ServerConnection(sqlConnection)); + server.Databases.ClearAndInitialize("[like(@Name, 'RDA%')]", new string[] { }); + ``` + +- Make Table.Location property optional for creating or scripting external tables. +- Enable scripting of ANSI_PADDING settings for Azure SQL Database tables. +- Remove obsolete types ServerActiveDirectory and DatabaseActiveDirectory +- Added BLOB_STORAGE scripting support for external data sources +- Fixed [error scripting external tables](https://feedback.azure.com/forums/908035-sql-server/suggestions/38267746-cannot-script-external-table-in-ssms-18-2) for Azure SQL Database +- Replace Microsoft.SqlServer.Management.SqlParser.dll with a dependency to its Nuget package + +## 160.1911221.0-preview + +- Increase major version from 15 to 16 +- Remove dependency on native batch parser from NetFx components +- Change NetStandard client driver to Microsoft.Data.SqlClient +- Add distribution property for DW materialized views +- Script FILLFACTOR for indexes on Azure SQL Database diff --git a/docs/media/certsettings.png b/docs/media/certsettings.png new file mode 100644 index 00000000..1832b767 Binary files /dev/null and b/docs/media/certsettings.png differ diff --git a/init.cmd b/init.cmd index 2807be83..db889800 100644 --- a/init.cmd +++ b/init.cmd @@ -29,7 +29,6 @@ doskey smoenum=pushd %BASEDIR%src\Microsoft\SqlServer\Management\SqlEnum\$* doskey clean=powershell.exe -ExecutionPolicy Unrestricted -File "%BASEDIR%init.ps1" -Clean doskey tst=pushd %BASEDIR%src\FunctionalTest\Smo\$* - REM == Common test command: doskey rtests=pushd %BASEDIR%target\distrib\debug\net462$Tvstest.console.exe microsoft.sqlserver.test.smo.dll /logger:trx /TestCaseFilter:"(TestCategory != Staging)" /Settings:%BASEDIR%src\FunctionalTest\Framework\functionaltest.runsettings $* doskey netcoretests=pushd %BASEDIR%target\distrib\debug\netcoreapp3.1$Tvstest.console.exe microsoft.sqlserver.test.smo.dll /logger:trx /TestCaseFilter:"(TestCategory != Staging)" /Settings:%BASEDIR%src\FunctionalTest\Framework\functionaltest.runsettings $* diff --git a/src/Codegen/cfg.xml b/src/Codegen/cfg.xml index ae9d58b0..3c5c3244 100644 --- a/src/Codegen/cfg.xml +++ b/src/Codegen/cfg.xml @@ -173,6 +173,7 @@ + diff --git a/src/Directory.Build.props b/src/Directory.Build.props index c851e54a..14e21075 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -89,7 +89,7 @@ - 160.22504.0 + 160.22510.0 Microsoft.Data.SqlClient 3.1.0 diff --git a/src/FunctionalTest/Smo/Agent/JobServerTests.cs b/src/FunctionalTest/Smo/Agent/JobServerTests.cs index 0f70369d..cc2f00e8 100644 --- a/src/FunctionalTest/Smo/Agent/JobServerTests.cs +++ b/src/FunctionalTest/Smo/Agent/JobServerTests.cs @@ -3,7 +3,6 @@ using System; using System.Data; -using System.Diagnostics; using System.Linq; using Microsoft.SqlServer.Management.Common; using Microsoft.SqlServer.Management.Sdk.Sfc; @@ -18,10 +17,10 @@ namespace Microsoft.SqlServer.Test.SMO.Agent { [TestClass] - [SupportedServerVersionRange(Edition = Management.Common.DatabaseEngineEdition.Enterprise)] public class JobServerTests : SqlTestBase { [TestMethod] + [SupportedServerVersionRange(Edition = Management.Common.DatabaseEngineEdition.Enterprise)] public void JobServer_PurgeJobHistory_generates_correct_query() { ExecuteTest(() => @@ -49,6 +48,7 @@ public void JobServer_PurgeJobHistory_generates_correct_query() [TestMethod] [UnsupportedHostPlatform(SqlHostPlatforms.Linux)] + [SupportedServerVersionRange(MinMajor = 16, Edition = DatabaseEngineEdition.Enterprise)] public void JobServer_EnumPerformanceCounters_returns_agent_counters() { ExecuteTest(() => @@ -70,6 +70,7 @@ public void JobServer_EnumPerformanceCounters_returns_agent_counters() } [TestMethod] + [SupportedServerVersionRange(Edition = Management.Common.DatabaseEngineEdition.Enterprise)] public void JobServer_Msx_methods_generate_correct_queries() { ExecuteTest(() => @@ -92,6 +93,8 @@ public void JobServer_Msx_methods_generate_correct_queries() } [TestMethod] + [SupportedServerVersionRange(Edition = Management.Common.DatabaseEngineEdition.Enterprise)] + public void JobServer_ErrorLog_methods_generate_correct_queries() { ExecuteTest(() => @@ -135,7 +138,9 @@ private Job CreateJob(string loginName = null) job.Refresh(); return job; } + [TestMethod] + [SupportedServerVersionRange(Edition = Management.Common.DatabaseEngineEdition.Enterprise)] public void JobServer_Job_methods() { ExecuteTest(() => @@ -180,6 +185,7 @@ public void JobServer_Job_methods() } [TestMethod] + [SupportedServerVersionRange(Edition = Management.Common.DatabaseEngineEdition.Enterprise)] public void JobServer_miscellaneous_methods_generate_correct_queries() { ExecuteTest(() => diff --git a/src/FunctionalTest/Smo/GeneralFunctionality/AuditSmoTests.cs b/src/FunctionalTest/Smo/GeneralFunctionality/AuditSmoTests.cs index 214a6f99..7b61094b 100644 --- a/src/FunctionalTest/Smo/GeneralFunctionality/AuditSmoTests.cs +++ b/src/FunctionalTest/Smo/GeneralFunctionality/AuditSmoTests.cs @@ -433,7 +433,7 @@ public void ServerAuditspecifications_Audit_Action_Types_In_DB_Should_Match_Enum UNION ALL SELECT name FROM (VALUES ('SELECT'),('UPDATE'),('INSERT'),('DELETE'),('EXECUTE'),('RECEIVE'),('REFERENCES')) actions(name)"; var dbAuditList = ServerContext.ConnectionContext.ExecuteWithResults(query).Tables[0].Rows.Cast().Select(row => row["name"].ToString()); - + System.Diagnostics.Trace.TraceInformation("Missing audit action types: {0}", string.Join(",", dbAuditList.Except(enumAttributeNames))); // enumAttributeNames is a superset of dbAuditList. Assert.That(enumAttributeNames, Is.SupersetOf(dbAuditList), @"Some types are missing in AuditActionType enum. Please, update enum AuditActionType in /src/Microsoft/SqlServer/Management/SqlEnum/enumstructs.cs"); diff --git a/src/FunctionalTest/Smo/GeneralFunctionality/DatabaseSmoTests.cs b/src/FunctionalTest/Smo/GeneralFunctionality/DatabaseSmoTests.cs index 215ef749..d2e328e9 100644 --- a/src/FunctionalTest/Smo/GeneralFunctionality/DatabaseSmoTests.cs +++ b/src/FunctionalTest/Smo/GeneralFunctionality/DatabaseSmoTests.cs @@ -119,19 +119,31 @@ public void EnumObjects_Sets_Synonym_Schema_And_Other_Properties() }; syn.Create(); + var script = database.ExecutionManager.RecordQueryText(() => + { + // Now, we retrieve the same object we just created by calling EnumObjects() + var objs = database.EnumObjects(DatabaseObjectTypes.Synonym, _SMO.SortOrder.Schema); + var synobj = objs.Rows.Cast().Where(r => (string)r["Name"] == expectedSynName).Single(); - // Now, we retrieve the same object we just created by calling EnumObjects() - var objs = database.EnumObjects(DatabaseObjectTypes.Synonym); - var synobj = objs.Rows.Cast().Where(r => (string)r["Name"] == expectedSynName).Single(); - - // The original bug was that Schema was coming back as blank, because there was an assumption - // that synonyms did not have a schema (which was incorrect) - Assert.That(synobj["Schema"], Is.EqualTo(expectedSynSchema), "Unexpected value for Schema"); + // The original bug was that Schema was coming back as blank, because there was an assumption + // that synonyms did not have a schema (which was incorrect) + Assert.That(synobj["Schema"], Is.EqualTo(expectedSynSchema), "Unexpected value for Schema"); - // While we are at it, let's also check the other properties... - Assert.That(synobj["DatabaseObjectTypes"], Is.EqualTo("Synonym"), "Unexpected value for DatabaseObjectTypes"); - Assert.That(synobj["Name"], Is.EqualTo(expectedSynName), "Unexpected value for Name"); - Assert.That(synobj["Urn"], Is.EqualTo(syn.Urn.ToString()), "Unexpected value for Urn"); + // While we are at it, let's also check the other properties... + Assert.That(synobj["DatabaseObjectTypes"], Is.EqualTo("Synonym"), "Unexpected value for DatabaseObjectTypes"); + Assert.That(synobj["Name"], Is.EqualTo(expectedSynName), "Unexpected value for Name"); + Assert.That(synobj["Urn"], Is.EqualTo(syn.Urn.ToString()), "Unexpected value for Urn"); + }, alsoExecute: true); + Assert.That(script.ToSingleString(), Contains.Substring("ORDER BY [Schema]"), "EnumObjects (SortOrder.Schema)"); + if (database.DatabaseEngineType == DatabaseEngineType.Standalone) + { + script = database.ExecutionManager.RecordQueryText(() => database.EnumObjects(), alsoExecute: true); + Assert.That(script.ToSingleString(), Contains.Substring("ORDER BY [DatabaseObjectTypes]"), "EnumObjects()"); + } + script = database.ExecutionManager.RecordQueryText(() => database.EnumObjects(DatabaseObjectTypes.Table, _SMO.SortOrder.Name), alsoExecute: true); + Assert.That(script.ToSingleString(), Contains.Substring("ORDER BY [Name]"), "EnumObjects()"); + script = database.ExecutionManager.RecordQueryText(() => database.EnumObjects(DatabaseObjectTypes.View, _SMO.SortOrder.Urn), alsoExecute: true); + Assert.That(script.ToSingleString(), Contains.Substring("ORDER BY [Urn]"), "EnumObjects()"); }); } diff --git a/src/FunctionalTest/Smo/GeneralFunctionality/DwSmoTests.cs b/src/FunctionalTest/Smo/GeneralFunctionality/DwSmoTests.cs index 1b847df3..fa507f56 100644 --- a/src/FunctionalTest/Smo/GeneralFunctionality/DwSmoTests.cs +++ b/src/FunctionalTest/Smo/GeneralFunctionality/DwSmoTests.cs @@ -65,7 +65,6 @@ public void Server_Databases_collection_enumerates_without_exception() } }); - Assert.That(databases.FirstOrDefault(d => d.EndsWith("Unknown")), Is.Not.Null, "Expected at least one Unknown engine edition. Make sure the test server has a paused DW instance. " + String.Join(",", databases)); }); } } diff --git a/src/FunctionalTest/Smo/GeneralFunctionality/PermissionsEnumTests.cs b/src/FunctionalTest/Smo/GeneralFunctionality/PermissionsEnumTests.cs new file mode 100644 index 00000000..5dea7719 --- /dev/null +++ b/src/FunctionalTest/Smo/GeneralFunctionality/PermissionsEnumTests.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Data; +using System.Globalization; +using System.Linq; +using System.Text; +using Microsoft.SqlServer.Management.Common; +using Microsoft.SqlServer.Management.Smo; +using Microsoft.SqlServer.Test.Manageability.Utils.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NUnit.Framework; +using Assert = NUnit.Framework.Assert; + +namespace Microsoft.SqlServer.Test.SMO.GeneralFunctionality +{ + [TestClass] + [UnsupportedDatabaseEngineEdition(DatabaseEngineEdition.SqlOnDemand)] + public class PermissionsEnumTests : SqlTestBase + { + [TestMethod] + public void PermEnum_DatabasePermissionSetValue_enum_is_complete() + { + ExecuteTest(() => + { + CompareEnumToServerPermissions(typeof(DatabasePermissionSetValue), @"select type, permission_name from sys.fn_builtin_permissions('DATABASE')"); + }); + } + + [TestMethod] + public void PermEnum_ServerPermissionSetValue_enum_is_complete() + { + ExecuteTest(() => + { + CompareEnumToServerPermissions(typeof(ServerPermissionSetValue), @"select type, permission_name from sys.fn_builtin_permissions('SERVER')"); + }); + } + + [TestMethod] + public void PermEnum_ObjectPermissionSetValue_enum_is_complete() + { + ExecuteTest(() => + { + CompareEnumToServerPermissions(typeof(ObjectPermissionSetValue), @"select type, permission_name from sys.fn_builtin_permissions(DEFAULT) where class_desc <> 'SERVER' and class_desc <> 'DATABASE'"); + }); + } + private void CompareEnumToServerPermissions(Type enumType, string permissionQuery) + { + var permissionTypes = GetAttributeValues(enumType); + var permissionNames = GetAttributeValues(enumType); + var permissionList = ServerContext.ConnectionContext.ExecuteWithResults(permissionQuery).Tables[0].Rows.Cast(); + var maxValue = Enum.GetValues(enumType).Cast().Max(); + var textInfo = new CultureInfo("en-US", false).TextInfo; + var missingValues = new StringBuilder(); + foreach (var row in permissionList) + { + var name = row["permission_name"].ToString().Trim(); + var type = row["type"].ToString().Trim(); + if (!permissionNames.Contains(name) || !permissionTypes.Contains(type)) + { + var enumName = new StringBuilder(); + foreach (var s in name.Split(' ')) + { + enumName.Append(textInfo.ToTitleCase(s.ToLowerInvariant())); + } + enumName.Append($" = {++maxValue},"); + missingValues.Append($"{Environment.NewLine}[PermissionType(\"{type}\")]{Environment.NewLine}[PermissionName(\"{name}\")]{Environment.NewLine}{enumName}{Environment.NewLine}"); + } + } + Assert.That(missingValues.ToString(), Is.Empty, $"{ enumType.Name} is incomplete. Add the missing values."); + } + /// + /// Returns the set of string attribute values associated with the given enumeration. + /// + /// + /// + /// + static internal IEnumerable GetAttributeValues(Type enumType) where T: StringValueAttribute { + return new HashSet(Enum.GetNames(enumType).Select(n => enumType.GetMember(n).Single().GetCustomAttributes(typeof(T), false).Cast().Single().Value)); + } + } +} diff --git a/src/FunctionalTest/Smo/ScriptingTests/Database_SmoTestSuite.cs b/src/FunctionalTest/Smo/ScriptingTests/Database_SmoTestSuite.cs index cc5c4421..9bc660dd 100644 --- a/src/FunctionalTest/Smo/ScriptingTests/Database_SmoTestSuite.cs +++ b/src/FunctionalTest/Smo/ScriptingTests/Database_SmoTestSuite.cs @@ -143,16 +143,17 @@ public void Trying_To_Access_AzureDB_With_DBManager_Role_Only_Does_Not_Throw() try { var pwd = SqlTestRandom.GeneratePassword(); - _SMO.Login login = null; + Login login = null; + User user = null; SqlConnection sqlConn = null; try { // Here we creata a login/user... login = master.Parent.CreateLogin(GenerateUniqueSmoObjectName("login_low_priv", maxLength: 128), _SMO.LoginType.SqlLogin, pwd); - var user = master.CreateUser(login.Name, login.Name); + user = master.CreateUser(login.Name, login.Name); - // ... and we grant it limite permissions (certainly a lot less than "cloudsa"). + // ... and we grant it limited permissions (certainly a lot less than "cloudsa"). // Note: the "dbmanager" role seems to only exist in SQL Azure, so for this reason // this test is restricted to that EngineType. master.ExecutionManager.ExecuteNonQuery( @@ -175,12 +176,10 @@ public void Trying_To_Access_AzureDB_With_DBManager_Role_Only_Does_Not_Throw() sqlConn.Open(); var dbScopedConn = new ServerConnection(sqlConn); var server = new _SMO.Server(dbScopedConn); - - // Before the fix, trying to do this would cause an exception. - Assert.DoesNotThrow(() => - { - var dbee = server.Databases.Count; - }, "Unexpected exception."); + server.SetDefaultInitFields(typeof(Database), nameof(Database.AzureEdition)); + Assert.That(server.Databases.Cast().Select(d => d.DatabaseEngineEdition), Has.None.EqualTo(DatabaseEngineEdition.Unknown), "Low privileged user should get DatabaseEngineEdition of all databases"); + Assert.That(server.Databases.Cast().Where(d => d.DatabaseEngineEdition == DatabaseEngineEdition.SqlDatabase).Select(d => d.AzureEdition), Has.None.EqualTo(""), "Low privileged user should get AzureEdition of all Azure DB databases"); + Assert.Throws(() => { var x = server.Databases[db.Name].Size; }, "Accessing other properties of inaccessible Database should throw"); } } finally @@ -195,7 +194,7 @@ public void Trying_To_Access_AzureDB_With_DBManager_Role_Only_Does_Not_Throw() master.ExecutionManager.ExecuteNonQuery( string.Format("ALTER ROLE dbmanager DROP member {0}", SmoObjectHelpers.SqlBracketQuoteString(login.Name))); - + user?.Drop(); login.Drop(); } } diff --git a/src/FunctionalTest/Smo/ScriptingTests/ExternalFileFormat_SmoTestSuite.cs b/src/FunctionalTest/Smo/ScriptingTests/ExternalFileFormat_SmoTestSuite.cs index 7a54c750..3126ee47 100644 --- a/src/FunctionalTest/Smo/ScriptingTests/ExternalFileFormat_SmoTestSuite.cs +++ b/src/FunctionalTest/Smo/ScriptingTests/ExternalFileFormat_SmoTestSuite.cs @@ -40,7 +40,7 @@ public void VerifyPositiveExternalFileFormatCreateDrop_AzureSterlingV12_SqlDW() /// server have the same values for their properties/scripts as expected. /// [VSTest.TestMethod] - [SupportedServerVersionRange(DatabaseEngineType = DatabaseEngineType.Standalone, MinMajor = 13, MaxMajor =15)] + [SupportedServerVersionRange(DatabaseEngineType = DatabaseEngineType.Standalone, MinMajor = 13, MaxMajor = 15)] [UnsupportedHostPlatform(SqlHostPlatforms.Linux)] [SqlTestArea(SqlTestArea.Polybase)] [UnsupportedDatabaseEngineEdition(DatabaseEngineEdition.SqlOnDemand)] @@ -49,6 +49,51 @@ public void VerifyPositiveExternalFileFormatCreateDrop_2016AndAfterOnPrem() VerifyPositiveExternalFileFormatCreateDrop(); } + /// + /// Verifies that SQL database objects created using SMO from a SQL On Demand + /// server have the same values for their properties/scripts as expected. + /// + [VSTest.DataTestMethod] + // positive unit tests for the DELIMITEDTEXT file format type + // supported DDL options are: (1) no optional properties; (2) any of the format properties (field terminator, string delimiter and use type default; + // NOTE: The serde method is not supported with the format type being DELIMITEDTEXT. + // 1. no optional properties + // 2. any of the format options + // 3. data compression + // 4. format options and data compression + [VSTest.DataRow("eff1", ExternalFileFormatType.DelimitedText, "", "", "", "", null, "", null)] + [VSTest.DataRow("eff1", ExternalFileFormatType.DelimitedText, "", "|", "", "", null, "", null)] + [VSTest.DataRow("eff1", ExternalFileFormatType.DelimitedText, "", "", "#", "", null, "", null)] + [VSTest.DataRow("eff1", ExternalFileFormatType.DelimitedText, "", "", "", "", false, "", null)] + [VSTest.DataRow("eff1", ExternalFileFormatType.DelimitedText, "", "", "", "", true, "", null)] + [VSTest.DataRow("eff1", ExternalFileFormatType.DelimitedText, "", "", "", "", null, "", 41)] + [VSTest.DataRow("eff1", ExternalFileFormatType.DelimitedText, "", "", "", "", null, "", 1)] + [VSTest.DataRow("eff1", ExternalFileFormatType.DelimitedText, "", "|", "#", "", false, "", 41)] + // positive unit tests for PARQUET file format type + // supported DDL options are: (1) no optional properties; (2) data compression property + // NOTE: The serde method and the format options are not supported with the format type being PARQUET. + // 1. no optional properties + // 2. data compression + [VSTest.DataRow("eff1", ExternalFileFormatType.Parquet, "", "", "", "", null, "", null)] + [VSTest.DataRow("eff1", ExternalFileFormatType.Parquet, "", "", "", "", null, "org.apache.hadoop.io.compress.GzipCodec", null)] + // positive unit tests for DELTA file format type + // supported DDL options are: (1) no optional properties; (2) data compression property + // NOTE: The serde method and the format options are not supported with the format type being DELTA. + // 1. no optional properties + // 2. data compression + [VSTest.DataRow("eff1", ExternalFileFormatType.Delta, "", "", "", "", null, "", null)] + [VSTest.DataRow("eff1", ExternalFileFormatType.Delta, "", "", "", "", null, "org.apache.hadoop.io.compress.GzipCodec", null)] + [SupportedServerVersionRange(Edition = DatabaseEngineEdition.SqlOnDemand)] + public void VerifyPositiveExternalFileFormatCreateDrop_SqlOnDemand(string name, ExternalFileFormatType type, string serDeMethod, string stringTerminator, string stringDelimiter, string dateFormat, bool? useTypeDefaultOption, string dataCompression, int? firstRow) + { + this.ExecuteWithDbDrop( + database => + { + VerifyPositiveExternalFileFormatCreateDropHelper(database, name, type, serDeMethod, stringTerminator, stringDelimiter, dateFormat, useTypeDefaultOption, dataCompression, firstRow); + } + ); + } + /// Tests creating, dropping and scripting of the external file format objects via SMO. /// Positive test steps: /// 1. Create an external file format with the format type property. @@ -132,6 +177,22 @@ private void VerifyPositiveExternalFileFormatCreateDrop(AzureDatabaseEdition azu azureDatabaseEdition); } + private void VerifyPositiveDeltaExternalFileFormatCreateDrop(string externalFileFormatName, string dataCompression) + { + this.ExecuteWithDbDrop( + database => + { + // positive unit tests for DELTA file format type + // supported DDL options are: (1) no optional properties; + // NOTE: The serde method and the format options are not supported with the format type being DELTA. + // 1. no optional properties + // 2. data compression + VerifyPositiveExternalFileFormatCreateDropHelper(database, externalFileFormatName, ExternalFileFormatType.Delta, string.Empty, string.Empty, string.Empty, string.Empty, null, dataCompression); + }, + AzureDatabaseEdition.NotApplicable + ); + } + /// /// Tests dropping an external file format with IF EXISTS option through SMO on SQL16 and later. /// @@ -284,7 +345,7 @@ private void VerifyPositiveExternalFileFormatCreateDropHelper( scriptTemplate.Append(string.Format(ExternalFileFormatScriptCreateTemplate, fullyFormatedNameForScripting, this.GetSqlKeywordForFileFormatType(externalFileFormat.FormatType))); // process optional parameters for each file format type and add them to the T-SQL script - ProcessOptionalProperties(externalFileFormat, scriptTemplate, db.DatabaseEngineEdition == DatabaseEngineEdition.SqlDataWarehouse); + ProcessOptionalProperties(externalFileFormat, scriptTemplate, db.DatabaseEngineEdition == DatabaseEngineEdition.SqlDataWarehouse || db.DatabaseEngineEdition == DatabaseEngineEdition.SqlOnDemand); TraceHelper.TraceInformation(createExternalFileFormatScripts); TraceHelper.TraceInformation(scriptTemplate.ToString()); Assert.That(createExternalFileFormatScripts, Does.Contain(scriptTemplate.ToString())); @@ -474,12 +535,6 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, string errorMessage = string.Empty; var ex = Assert.Throws(externalFileFormat.Create, "verify the external file format was not created due to unset FormatType"); - var innerEx = ex.GetBaseException(); - Assert.Multiple(() => - { - Assert.That(innerEx, Is.InstanceOf(), "innermost exception"); - Assert.That(innerEx.Message, Does.Contain(nameof(externalFileFormat.FormatType)), "innermost exception message"); - }); // // Step 2. Create external file format with conflicting properties - format type is DelimitedText and SerDeMethod. @@ -487,24 +542,8 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.FormatType = externalFileFormatType; externalFileFormat.SerDeMethod = externalFileFormatSerDeMethod; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "SerDeMethod", externalFileFormat.SerDeMethod, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); + Assert.Throws(externalFileFormat.Create); - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } // // Step 3. Create external file format with conflicting properties - format type is RcFile and FieldTerminator. @@ -512,24 +551,8 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.FormatType = ExternalFileFormatType.RcFile; externalFileFormat.FieldTerminator = externalFileFormatFieldTerminator; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "FieldTerminator", externalFileFormat.FieldTerminator, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); + Assert.Throws(externalFileFormat.Create); - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } // // Step 4. Create external file format with conflicting properties - format type is RcFile and StringDelimiter. @@ -538,24 +561,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.FieldTerminator = string.Empty; externalFileFormat.StringDelimiter = externalFileFormatStringDelimiter; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "StringDelimiter", externalFileFormat.StringDelimiter, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); // // Step 5. Create external file format with conflicting properties - format type is RcFile and DateFormat. @@ -565,24 +571,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.StringDelimiter = string.Empty; externalFileFormat.DateFormat = externalFileFormatDateFormat; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "DateFormat", externalFileFormat.DateFormat, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); // // Step 6. Create external file format with conflicting properties - format type is RcFile and UseTypeDefault. @@ -593,26 +582,9 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.DateFormat = string.Empty; externalFileFormat.UseTypeDefault = true; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "UseTypeDefault", externalFileFormat.UseTypeDefault.ToString(), externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); - if (db.DatabaseEngineEdition == DatabaseEngineEdition.SqlDataWarehouse) + if (db.DatabaseEngineEdition == DatabaseEngineEdition.SqlDataWarehouse || db.DatabaseEngineEdition == DatabaseEngineEdition.SqlOnDemand) { // // Step 7. Create external file format with conflicting properties - format type is RcFile and FirstRow. @@ -624,24 +596,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.UseTypeDefault = false; externalFileFormat.FirstRow = 10; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "FirstRow", externalFileFormat.FirstRow, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); } // @@ -650,24 +605,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.FormatType = ExternalFileFormatType.Orc; externalFileFormat.SerDeMethod = externalFileFormatSerDeMethod; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "SerDeMethod", externalFileFormat.SerDeMethod, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); // // Step 9. Create external file format with conflicting properties - format type is Orc and FieldTerminator. @@ -676,24 +614,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.SerDeMethod = string.Empty; externalFileFormat.FieldTerminator = externalFileFormatFieldTerminator; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "FieldTerminator", externalFileFormat.FieldTerminator, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); // // Step 10. Create external file format with conflicting properties - format type is Orc and StringDelimiter. @@ -703,24 +624,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.FieldTerminator = string.Empty; externalFileFormat.StringDelimiter = externalFileFormatStringDelimiter; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "StringDelimiter", externalFileFormat.StringDelimiter, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); // // Step 11. Create external file format with conflicting properties - format type is Orc and DateFormat. @@ -731,24 +635,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.StringDelimiter = string.Empty; externalFileFormat.DateFormat = externalFileFormatDateFormat; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "DateFormat", externalFileFormat.DateFormat, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); // // Step 12. Create external file format with conflicting properties - format type is Orc and UseTypeDefault. @@ -760,26 +647,9 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.DateFormat = string.Empty; externalFileFormat.UseTypeDefault = true; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "UseTypeDefault", externalFileFormat.UseTypeDefault, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); - if (db.DatabaseEngineEdition == DatabaseEngineEdition.SqlDataWarehouse) + if (db.DatabaseEngineEdition == DatabaseEngineEdition.SqlDataWarehouse || db.DatabaseEngineEdition == DatabaseEngineEdition.SqlOnDemand) { // // Step 13. Create external file format with conflicting properties - format type is Orc and FirstRow. @@ -792,24 +662,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.UseTypeDefault = false; externalFileFormat.FirstRow = 10; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "FirstRow", externalFileFormat.FirstRow, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); } // @@ -818,24 +671,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.FormatType = ExternalFileFormatType.Parquet; externalFileFormat.SerDeMethod = externalFileFormatSerDeMethod; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "SerDeMethod", externalFileFormat.SerDeMethod, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); // // Step 15. Create external file format with conflicting properties - format type is Parquet and FieldTerminator. @@ -844,24 +680,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.SerDeMethod = string.Empty; externalFileFormat.FieldTerminator = externalFileFormatFieldTerminator; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "FieldTerminator", externalFileFormat.FieldTerminator, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); // // Step 16. Create external file format with conflicting properties - format type is Parquet and StringDelimiter. @@ -871,24 +690,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.FieldTerminator = string.Empty; externalFileFormat.StringDelimiter = externalFileFormatStringDelimiter; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "StringDelimiter", externalFileFormat.StringDelimiter, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); // // Step 17. Create external file format with conflicting properties - format type Parquet and DateFormat. @@ -899,24 +701,7 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.StringDelimiter = string.Empty; externalFileFormat.DateFormat = externalFileFormatDateFormat; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "DateFormat", externalFileFormat.DateFormat, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); // // Step 18. Create external file format with conflicting properties - format type is Parquet and UseTypeDefault. @@ -928,26 +713,9 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.DateFormat = string.Empty; externalFileFormat.UseTypeDefault = true; - // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "UseTypeDefault", externalFileFormat.UseTypeDefault, externalFileFormat.FormatType.ToString()); - - externalFileFormat.Create(); - - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) - { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } + Assert.Throws(externalFileFormat.Create); - if (db.DatabaseEngineEdition == DatabaseEngineEdition.SqlDataWarehouse) + if (db.DatabaseEngineEdition == DatabaseEngineEdition.SqlDataWarehouse || db.DatabaseEngineEdition == DatabaseEngineEdition.SqlOnDemand) { // // Step 19. Create external file format with conflicting properties - format type is Parquet and FirstRow. @@ -961,24 +729,137 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, externalFileFormat.FirstRow = 10; // verify the external file format was not created - try - { - // attempt to create an external file format - errorMessage = string.Format("Cannot set the property '{0}' to '{1}' because the property '{0}' is not supported for external file format type '{2}'.", "FirstRow", externalFileFormat.FirstRow, externalFileFormat.FormatType.ToString()); + errorMessage = $"Cannot set the property 'FirstRow' to '{externalFileFormat.FirstRow}' because the property 'FirstRow' is not supported for external file format type '{externalFileFormat.FormatType.ToString()}'."; + + // attempt to create an external file format + Assert.Throws(externalFileFormat.Create); + } + } + + /// + /// Tests negative scenarios for serverless database objects created using SMO from a SQL On Prem + /// server. + /// + [VSTest.TestMethod] + [SupportedServerVersionRange(Edition = DatabaseEngineEdition.SqlOnDemand)] + public void VerifyNegativeExternalFileFormatCreateDrop_SqlOnDemand() + { + VerifyNegativeDeltaExternalFileFormatCreateDrop(); + VerifyNegativeExternalFileFormatCreateDrop(); + } + + /// Tests creating, dropping and scripting of the Delta external file format objects via SMO. + /// Negative test steps: + /// 1. Create external file format with no required properties. + /// 2. Create external file format with conflicting properties - format type is Delta and SerDeMethod. + /// 3. Create external file format with conflicting properties - format type is Delta and FieldTerminator. + /// 4. Create external file format with conflicting properties - format type is Delta and StringDelimiter. + /// 5. Create external file format with conflicting properties - format type Delta and DateFormat. + /// 6. Create external file format with conflicting properties - format type is Delta and UseTypeDefault. + /// 7. Create external file format with conflicting properties - format type is Delta and FirstRow. + private void VerifyNegativeDeltaExternalFileFormatCreateDrop() + { + // const definitions + const string ExternalFileFormatSerdeMethod = @"org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"; + const string FieldTerminator = "|"; + const string StringDelimiter = "#"; + const string DateFormat = "MM-dd-yyyy"; - externalFileFormat.Create(); + string externalFileFormatName = "eff1"; - // validate expected exception and error message - Assert.Fail(errorMessage, externalFileFormat.Name); - } - catch (SmoException e) + this.ExecuteWithDbDrop( + database => { - if (!ExceptionHelpers.IsExpectedException(e, errorMessage)) - { - throw; - } - } - } + // negative unit tests for the DELTA file format type + // unsupported DDL options are: serde method, + VerifyNegativeDeltaExternalFileFormatCreateDropHelper(database, externalFileFormatName, ExternalFileFormatSerdeMethod, FieldTerminator, StringDelimiter, DateFormat); + }, + AzureDatabaseEdition.NotApplicable); + } + + /// + /// Executes negative tests for the create delta external file format object. + /// + /// The database name. + /// The external file format name. + /// The external file format serialize/deserialize method property value. + /// The external file format field terminator property value. + /// The external file format string delimiter property value. + /// The external file format date format property value. + private void VerifyNegativeDeltaExternalFileFormatCreateDropHelper(Database db, + string externalFileFormatName, + string externalFileFormatSerDeMethod, + string externalFileFormatFieldTerminator, + string externalFileFormatStringDelimiter, + string externalFileFormatDateFormat) + { + + const string ExternalFileFormatTestName = "Delta External File Format Testing"; + + // + // Step 1. Create external file format with no required properties. + // + TraceHelper.TraceInformation("Step 1: {0} - Creating external file format {1} with no required properties.", ExternalFileFormatTestName, externalFileFormatName); + ExternalFileFormat externalFileFormat = new ExternalFileFormat(db, externalFileFormatName); + + string errorMessage = string.Empty; + + var ex = Assert.Throws(externalFileFormat.Create, "verify the external file format was not created due to unset FormatType"); + + // + // Step 2. Create external file format with conflicting properties - format type is Delta and SerDeMethod. + // + externalFileFormat.FormatType = ExternalFileFormatType.Delta; + externalFileFormat.SerDeMethod = externalFileFormatSerDeMethod; + + // attempt to create an external file format + ex = Assert.Throws(externalFileFormat.Create); + + // + // Step 3. Create external file format with conflicting properties - format type is Delta and FieldTerminator. + // + externalFileFormat.FormatType = ExternalFileFormatType.Delta; + externalFileFormat.SerDeMethod = string.Empty; + externalFileFormat.FieldTerminator = externalFileFormatFieldTerminator; + + // attempt to create an external file format + ex = Assert.Throws(externalFileFormat.Create); + + // + // Step 4. Create external file format with conflicting properties - format type is Delta and StringDelimiter. + // + externalFileFormat.FormatType = ExternalFileFormatType.Delta; + externalFileFormat.SerDeMethod = string.Empty; + externalFileFormat.FieldTerminator = string.Empty; + externalFileFormat.StringDelimiter = externalFileFormatStringDelimiter; + + // attempt to create an external file format + ex = Assert.Throws(externalFileFormat.Create); + + // + // Step 5. Create external file format with conflicting properties - format type Delta and DateFormat. + // + externalFileFormat.FormatType = ExternalFileFormatType.Delta; + externalFileFormat.SerDeMethod = string.Empty; + externalFileFormat.FieldTerminator = string.Empty; + externalFileFormat.StringDelimiter = string.Empty; + externalFileFormat.DateFormat = externalFileFormatDateFormat; + + // attempt to create an external file format + ex = Assert.Throws(externalFileFormat.Create); + + // + // Step 6. Create external file format with conflicting properties - format type is Delta and UseTypeDefault. + // + externalFileFormat.FormatType = ExternalFileFormatType.Delta; + externalFileFormat.SerDeMethod = string.Empty; + externalFileFormat.FieldTerminator = string.Empty; + externalFileFormat.StringDelimiter = string.Empty; + externalFileFormat.DateFormat = string.Empty; + externalFileFormat.UseTypeDefault = true; + + // attempt to create an external file format + ex = Assert.Throws(externalFileFormat.Create); } /// @@ -987,17 +868,18 @@ private void VerifyNegativeExternalFileFormatCreateDropHelper(Database db, /// /// /// - /// /// True if SQL DW DB. - private void ProcessOptionalProperties(ExternalFileFormat externalFileFormat, StringBuilder script, bool isSqlDw) + /// /// True if SQL DW or Serverless DB. + private void ProcessOptionalProperties(ExternalFileFormat externalFileFormat, StringBuilder script, bool isSqlDwOrServerless) { switch (externalFileFormat.FormatType) { case ExternalFileFormatType.DelimitedText: - ValidateDelimitedTextProperties(externalFileFormat, script, isSqlDw); + ValidateDelimitedTextProperties(externalFileFormat, script, isSqlDwOrServerless); break; case ExternalFileFormatType.Orc: case ExternalFileFormatType.Parquet: - ValidateOrcOrParquetProperties(externalFileFormat, script); + case ExternalFileFormatType.Delta: + ValidateOrcParquetOrDeltaProperties(externalFileFormat, script); break; case ExternalFileFormatType.RcFile: ValidateRcFileProperties(externalFileFormat, script); @@ -1014,18 +896,18 @@ private void ProcessOptionalProperties(ExternalFileFormat externalFileFormat, St /// /// External file format. /// The external file format T-SQL script. - /// True if SQL DW DB. - private void ValidateDelimitedTextProperties(ExternalFileFormat externalFileFormat, StringBuilder script, bool isSqlDw) + /// True if SQL DW or Serverless DB. + private void ValidateDelimitedTextProperties(ExternalFileFormat externalFileFormat, StringBuilder script, bool isSqlDwOrServerless) { // check for optional properties StringBuilder formatOptions = new StringBuilder(); // if format options optional properties are specified, they need to be enclosed in FORMAT_OPTIONS() - VerifyOptionalFormatParameters(externalFileFormat, externalFileFormat.FieldTerminator, "FIELD_TERMINATOR = N'{0}'", formatOptions); - VerifyOptionalFormatParameters(externalFileFormat, externalFileFormat.StringDelimiter, "STRING_DELIMITER = N'{0}'", formatOptions); - VerifyOptionalFormatParameters(externalFileFormat, externalFileFormat.DateFormat, "DATE_FORMAT = N'{0}'", formatOptions); + VerifyOptionalFormatParameters(externalFileFormat.FieldTerminator, "FIELD_TERMINATOR = N'{0}'", formatOptions); + VerifyOptionalFormatParameters(externalFileFormat.StringDelimiter, "STRING_DELIMITER = N'{0}'", formatOptions); + VerifyOptionalFormatParameters(externalFileFormat.DateFormat, "DATE_FORMAT = N'{0}'", formatOptions); - if (isSqlDw && externalFileFormat.FirstRow > 1) + if (isSqlDwOrServerless && externalFileFormat.FirstRow > 1) { if (formatOptions.Length > 0) { @@ -1048,7 +930,7 @@ private void ValidateDelimitedTextProperties(ExternalFileFormat externalFileForm script.Append(string.Format(", FORMAT_OPTIONS ({0})", fileFormatOptions)); } - VerifyOptionalFormatParameters(externalFileFormat, externalFileFormat.DataCompression, "DATA_COMPRESSION = N'{0}'", script); + VerifyOptionalFormatParameters(externalFileFormat.DataCompression, "DATA_COMPRESSION = N'{0}'", script); } /// @@ -1057,10 +939,10 @@ private void ValidateDelimitedTextProperties(ExternalFileFormat externalFileForm /// /// External file format. /// The external file format T-SQL script. - private void ValidateOrcOrParquetProperties(ExternalFileFormat externalFileFormat, StringBuilder script) + private void ValidateOrcParquetOrDeltaProperties(ExternalFileFormat externalFileFormat, StringBuilder script) { // check for optional properties - VerifyOptionalFormatParameters(externalFileFormat, externalFileFormat.DataCompression, "DATA_COMPRESSION = N'{0}'", script); + VerifyOptionalFormatParameters(externalFileFormat.DataCompression, "DATA_COMPRESSION = N'{0}'", script); } /// @@ -1072,20 +954,19 @@ private void ValidateOrcOrParquetProperties(ExternalFileFormat externalFileForma private void ValidateRcFileProperties(ExternalFileFormat externalFileFormat, StringBuilder script) { // check for optional properties - VerifyOptionalFormatParameters(externalFileFormat, externalFileFormat.SerDeMethod, "SERDE_METHOD = N'{0}'", script); + VerifyOptionalFormatParameters(externalFileFormat.SerDeMethod, "SERDE_METHOD = N'{0}'", script); - VerifyOptionalFormatParameters(externalFileFormat, externalFileFormat.DataCompression, "DATA_COMPRESSION = N'{0}'", script); + VerifyOptionalFormatParameters(externalFileFormat.DataCompression, "DATA_COMPRESSION = N'{0}'", script); } /// /// Verifies format options optional properties and adds a comma to the generated script /// when more than one optional format properties exist. /// - /// External file format. /// The format options property value. /// The T-SQL script to add to the format options script. /// The T-SQL format options script. - private void VerifyOptionalFormatParameters(ExternalFileFormat externalFileFormat, string propertyValue, string sqlScript, StringBuilder formatOptions) + private void VerifyOptionalFormatParameters(string propertyValue, string sqlScript, StringBuilder formatOptions) { if (!string.IsNullOrEmpty(propertyValue)) { @@ -1114,6 +995,8 @@ private string GetSqlKeywordForFileFormatType(ExternalFileFormatType fileFormatT return "PARQUET"; case ExternalFileFormatType.RcFile: return "RCFILE"; + case ExternalFileFormatType.Delta: + return "DELTA"; default: Assert.AreEqual(ExternalFileFormatType.DelimitedText, fileFormatType); return "DELIMITEDTEXT"; diff --git a/src/FunctionalTest/Smo/ToolsConnectionInfo.xml b/src/FunctionalTest/Smo/ToolsConnectionInfo.xml index 0d5a7b54..0921c638 100644 --- a/src/FunctionalTest/Smo/ToolsConnectionInfo.xml +++ b/src/FunctionalTest/Smo/ToolsConnectionInfo.xml @@ -178,4 +178,3 @@ Sql2012;Sql2017;SQL2017Linux passwordCredential='saPassword-2017Linux' db_engine_edition="Enterprise"/> - diff --git a/src/FunctionalTest/Smo/XEvent/TargetColumnInfoUnitTest.cs b/src/FunctionalTest/Smo/XEvent/TargetColumnInfoUnitTest.cs index cbb5e314..e89461a8 100644 --- a/src/FunctionalTest/Smo/XEvent/TargetColumnInfoUnitTest.cs +++ b/src/FunctionalTest/Smo/XEvent/TargetColumnInfoUnitTest.cs @@ -115,7 +115,7 @@ public void TestTargetColumnCollectionContains() public void TestTargetColumnInfoSet() { var columns = store.EventFileTargetInfo.TargetColumnInfoSet.OfType().Select(c => c.Name); - Assert.That(columns, Is.EquivalentTo(new string[] { "external_telemetry_query", "filename", "increment", "is_indexed_file_target", "lazy_create_blob", "max_file_size", "max_rollover_files", "metadatafile"}), + Assert.That(columns, Is.EquivalentTo(new string[] { "add_app_name, external_telemetry_query", "filename", "increment", "is_indexed_file_target", "lazy_create_blob", "max_file_size", "max_rollover_files", "metadatafile"}), "Unexpected columns for file target"); } } diff --git a/src/FunctionalTest/Smo/XEvent/TargetUnitTest.cs b/src/FunctionalTest/Smo/XEvent/TargetUnitTest.cs index 1625ca15..eb872dde 100644 --- a/src/FunctionalTest/Smo/XEvent/TargetUnitTest.cs +++ b/src/FunctionalTest/Smo/XEvent/TargetUnitTest.cs @@ -244,7 +244,7 @@ public void TestTargetGetTargetFields() Assert.IsNotNull(target.TargetFields); //package0.event_file has 5 customizable fields //filename is set to "file" in this session - Assert.That(target.TargetFields.Count, Is.EqualTo(8), "TestTargetGetTargetFields event_file TargetFields.Count" ); // metadatafile is removed + Assert.That(target.TargetFields.Count, Is.EqualTo(9), "TestTargetGetTargetFields event_file TargetFields.Count" ); Assert.AreEqual("https://account.blob.core.windows.net/container/filename.xel", target.TargetFields["filename"].Value); //other fields are all in default value diff --git a/src/Microsoft/SqlServer/Management/ConnectionInfo/ConnectionEnums.cs b/src/Microsoft/SqlServer/Management/ConnectionInfo/ConnectionEnums.cs index c9d16552..56cef8e9 100644 --- a/src/Microsoft/SqlServer/Management/ConnectionInfo/ConnectionEnums.cs +++ b/src/Microsoft/SqlServer/Management/ConnectionInfo/ConnectionEnums.cs @@ -119,6 +119,10 @@ make sure to update both enums with any changes [CommonDisplayNameKey("SqlDatabaseEdgeEdition")] SqlDatabaseEdge = 0x000009, + ///The server is an Azure Arc Managed SQL Instance Edition + [CommonDisplayNameKey("SqlAzureArcManagedInstanceEdition")] + SqlAzureArcManagedInstance = 0x00000A, + ///The server is Sql SqlOnDemand [CommonDisplayNameKey("SqlOnDemandEdition")] SqlOnDemand = 0x00000B, @@ -373,6 +377,7 @@ public static IEnumerable GetSupportedDatabaseEngineEditi yield return DatabaseEngineEdition.Standard; yield return DatabaseEngineEdition.SqlStretchDatabase; yield return DatabaseEngineEdition.SqlManagedInstance; + yield return DatabaseEngineEdition.SqlAzureArcManagedInstance; yield return DatabaseEngineEdition.SqlDatabaseEdge; break; default: diff --git a/src/Microsoft/SqlServer/Management/ConnectionInfo/StringConnectionInfo.strings b/src/Microsoft/SqlServer/Management/ConnectionInfo/StringConnectionInfo.strings index b6153b08..acaef538 100644 --- a/src/Microsoft/SqlServer/Management/ConnectionInfo/StringConnectionInfo.strings +++ b/src/Microsoft/SqlServer/Management/ConnectionInfo/StringConnectionInfo.strings @@ -53,6 +53,7 @@ StretchEdition = Microsoft SQL Server Stretch Database Edition SqlManagedInstanceEdition = Microsoft Azure SQL Database Managed Instance Edition SqlOnDemandEdition = Microsoft Azure Synapse SQL Analytics on-demand Edition SqlDatabaseEdgeEdition = Microsoft Azure SQL Edge Edition +SqlAzureArcManagedInstanceEdition = Microsoft Azure Arc SQL Managed Instance Edition # # Database Engine Type Names diff --git a/src/Microsoft/SqlServer/Management/Sdk/Sfc/Enumerator/XmlRead.cs b/src/Microsoft/SqlServer/Management/Sdk/Sfc/Enumerator/XmlRead.cs index 23f95a5c..5249e8f8 100644 --- a/src/Microsoft/SqlServer/Management/Sdk/Sfc/Enumerator/XmlRead.cs +++ b/src/Microsoft/SqlServer/Management/Sdk/Sfc/Enumerator/XmlRead.cs @@ -1644,7 +1644,7 @@ override public bool Next() /// ///class to read a special query [ComVisible(false)] - public class XmlReadSpecialQuery : XmlRead + public class XmlReadSpecialQuery : XmlReadConditionedStatement { /// ///initialize with reader @@ -1661,9 +1661,13 @@ public string Database /// ///get attribute - query - public String Query + public string Query { - get { return this.Reader["query"]; } + get { return Sql; } + } + public override bool Next() + { + return Next("special_query"); } } } diff --git a/src/Microsoft/SqlServer/Management/Smo/DatabaseBase.cs b/src/Microsoft/SqlServer/Management/Smo/DatabaseBase.cs index cb7ba288..fcec4e2f 100644 --- a/src/Microsoft/SqlServer/Management/Smo/DatabaseBase.cs +++ b/src/Microsoft/SqlServer/Management/Smo/DatabaseBase.cs @@ -27,9 +27,9 @@ namespace Microsoft.SqlServer.Management.Smo { [Facets.EvaluationMode(Dmf.AutomatedPolicyEvaluationMode.CheckOnSchedule)] - [Microsoft.SqlServer.Management.Sdk.Sfc.PhysicalFacet] - public partial class Database : ScriptNameObjectBase, Cmn.ICreatable, Cmn.IAlterable, Cmn.IDroppable, Cmn.IDropIfExists, - Cmn.ISafeRenamable, IExtendedProperties, IScriptable, IDatabaseOptions + [PhysicalFacet] + public partial class Database : ScriptNameObjectBase, ICreatable, IAlterable, IDroppable, IDropIfExists, + ISafeRenamable, IExtendedProperties, IScriptable, IDatabaseOptions { internal Database(AbstractCollectionBase parentColl, ObjectKeyBase key, SqlSmoState state) : base(parentColl, key, state) @@ -419,7 +419,9 @@ internal override void ScriptCreate(StringCollection createQuery, ScriptingPrefe var emptyFileGroups = new StringCollection(); var isAzureDb = Cmn.DatabaseEngineType.SqlAzureDatabase == sp.TargetDatabaseEngineType; var bSuppressDirtyCheck = sp.SuppressDirtyCheck; - var targetEditionIsManagedServer = !isAzureDb && sp.TargetDatabaseEngineEdition == Cmn.DatabaseEngineEdition.SqlManagedInstance; + var targetEditionIsManagedServer = !isAzureDb && + (sp.TargetDatabaseEngineEdition == Cmn.DatabaseEngineEdition.SqlManagedInstance || + sp.TargetDatabaseEngineEdition == Cmn.DatabaseEngineEdition.SqlAzureArcManagedInstance); if (IsSupportedProperty("DatabaseSnapshotBaseName")) { @@ -1102,8 +1104,9 @@ private void AddCompatibilityLevel(StringCollection query, ScriptingPreferences Property propCompat = Properties.Get("CompatibilityLevel"); if (null != propCompat.Value && (propCompat.Dirty || !sp.ScriptForAlter)) { - bool isTargetSqlAzureOrMI = (sp.TargetDatabaseEngineType == DatabaseEngineType.SqlAzureDatabase) || - (sp.TargetDatabaseEngineEdition == DatabaseEngineEdition.SqlManagedInstance); + bool isTargetSqlAzureOrMIOrMIAA = (sp.TargetDatabaseEngineType == DatabaseEngineType.SqlAzureDatabase) || + (sp.TargetDatabaseEngineEdition == DatabaseEngineEdition.SqlManagedInstance) || + (sp.TargetDatabaseEngineEdition == DatabaseEngineEdition.SqlAzureArcManagedInstance); bool isVersion160WithCompatLevelLessThan160 = (sp.TargetServerVersionInternal == SqlServerVersionInternal.Version160) && @@ -1156,11 +1159,11 @@ private void AddCompatibilityLevel(StringCollection query, ScriptingPreferences isVersion140WithCompatLevelLessThan140 || isVersion150WithCompatLevelLessThan150 || isVersion160WithCompatLevelLessThan160 || - isTargetSqlAzureOrMI; + isTargetSqlAzureOrMIOrMIAA; //script only if compatibility level is less than the target server // on Alter() we just script it and let the server fail if it is not correct - if (IsSupportedProperty("CompatibilityLevel", sp) && (sp.ScriptForAlter || isVersionWithLowerCompatLevel || isVersion80Or90WithLowerCompatLevel || isTargetSqlAzureOrMI)) + if (IsSupportedProperty("CompatibilityLevel", sp) && (sp.ScriptForAlter || isVersionWithLowerCompatLevel || isVersion80Or90WithLowerCompatLevel)) { CompatibilityLevel upgradedCompatLevel = UpgradeCompatibilityValueIfRequired(sp, (CompatibilityLevel)propCompat.Value); if (isVersionWithLowerCompatLevel) @@ -1863,7 +1866,7 @@ private void ContainmentRelatedValidation(ScriptingPreferences sp) //If containment supported on source database, check the version and enginetype of target. if (this.IsSupportedProperty("ContainmentType")) { - ContainmentType cType = this.GetPropValueOptional("ContainmentType", ContainmentType.None); + ContainmentType cType = this.GetPropValueOptional("ContainmentType", ContainmentType.None); if (cType == ContainmentType.None) { @@ -2529,11 +2532,11 @@ public System.Boolean DatabaseOwnershipChaining /// Gets or sets the MD Catalog Collation type. Only valid during creation, and we cannot specify ContainedDatabaseCollation explicitly /// [SfcProperty(SfcPropertyFlags.ReadOnlyAfterCreation | SfcPropertyFlags.SqlAzureDatabase)] - public Microsoft.SqlServer.Management.Smo.CatalogCollationType CatalogCollation + public CatalogCollationType CatalogCollation { get { - return (Microsoft.SqlServer.Management.Smo.CatalogCollationType)this.Properties.GetValueWithNullReplacement("CatalogCollation"); + return (CatalogCollationType)this.Properties.GetValueWithNullReplacement("CatalogCollation"); } set { @@ -5809,7 +5812,7 @@ public UrnInfo(string urnType, bool hasSchema, bool hasName, DatabaseObjectTypes } /// - /// The ListObjects method returns a SQLObjectList object that enumerates the system and user-defined objects defining the database referenced. + /// The EnumObjects method returns a DataTable that enumerates the system and user-defined objects defining the database referenced. /// /// /// @@ -5931,10 +5934,10 @@ public DataTable EnumObjects(DatabaseObjectTypes types, SortOrder order) finalQuery.Append(" ORDER BY "); switch (order) { - case SortOrder.Name: finalQuery.Append("Name"); break; - case SortOrder.Schema: finalQuery.Append("Schema"); break; - case SortOrder.Type: finalQuery.Append("DatabaseObjectTypes"); break; - default: finalQuery.Append("Urn"); break; + case SortOrder.Name: finalQuery.Append("[Name]"); break; + case SortOrder.Schema: finalQuery.Append("[Schema]"); break; + case SortOrder.Type: finalQuery.Append("[DatabaseObjectTypes]"); break; + default: finalQuery.Append("[Urn]"); break; } finalQuery.Append("\ndrop table #t"); @@ -5996,9 +5999,9 @@ public void TruncateLog() /// If true this means only header and body are needed, otherwise all properties /// internal static string[] GetScriptFields(Type parentType, - Cmn.ServerVersion version, - Cmn.DatabaseEngineType databaseEngineType, - Cmn.DatabaseEngineEdition databaseEngineEdition, + ServerVersion version, + DatabaseEngineType databaseEngineType, + DatabaseEngineEdition databaseEngineEdition, bool defaultTextMode) { string[] fields = @@ -6012,12 +6015,12 @@ internal static string[] GetScriptFields(Type parentType, nameof(IsLedger), nameof(PersistentVersionStoreFileGroup) }; - List list = GetSupportedScriptFields(typeof(Database.PropertyMetadataProvider),fields, version, databaseEngineType, databaseEngineEdition); + List list = GetSupportedScriptFields(typeof(PropertyMetadataProvider),fields, version, databaseEngineType, databaseEngineEdition); return list.ToArray(); } - internal static string[] GetScriptFields2(Type parentType, Cmn.ServerVersion version, - Cmn.DatabaseEngineType databaseEngineType, Cmn.DatabaseEngineEdition databaseEngineEdition, + internal static string[] GetScriptFields2(Type parentType, ServerVersion version, + DatabaseEngineType databaseEngineType, DatabaseEngineEdition databaseEngineEdition, bool defaultTextMode, ScriptingPreferences sp) { string[] fields = @@ -6026,7 +6029,7 @@ internal static string[] GetScriptFields2(Type parentType, Cmn.ServerVersion ver "IsMirroringEnabled", "IsVarDecimalStorageFormatEnabled", }; - List list = GetSupportedScriptFields(typeof(Database.PropertyMetadataProvider), fields, version, databaseEngineType, databaseEngineEdition); + List list = GetSupportedScriptFields(typeof(PropertyMetadataProvider), fields, version, databaseEngineType, databaseEngineEdition); return list.ToArray(); } @@ -6039,9 +6042,9 @@ public System.Boolean IsVarDecimalStorageFormatSupported { // vardecimal is supported in SQL Server 2005, SP2 and later, for Enterprise Edition only. // vardecimal will be replaced by a different compression feature in Katmai - System.Version yukonSp2 = new System.Version(9, 0, 3003); + Version yukonSp2 = new Version(9, 0, 3003); - System.Version thisversion = new System.Version( + Version thisversion = new Version( this.Parent.ConnectionContext.ServerVersion.Major, this.Parent.ConnectionContext.ServerVersion.Minor, this.Parent.ConnectionContext.ServerVersion.BuildNumber); @@ -6056,6 +6059,7 @@ public System.Boolean IsVarDecimalStorageFormatSupported return (thisversion > yukonSp2) && (this.Parent.Information.EngineEdition == Edition.EnterpriseOrDeveloper || this.Parent.Information.EngineEdition == Edition.SqlManagedInstance || + this.Parent.Information.EngineEdition == Edition.SqlAzureArcManagedInstance || this.Parent.Information.EngineEdition == Edition.SqlDatabaseEdge); } } @@ -6386,7 +6390,9 @@ public void InitTableColumns() } private void ScriptDbOptionsProps(StringCollection query, ScriptingPreferences sp, bool isAzureDb) { - var targetEditionIsManagedServer = sp.TargetDatabaseEngineEdition == Cmn.DatabaseEngineEdition.SqlManagedInstance; + var targetEditionIsManagedServer = + ((sp.TargetDatabaseEngineEdition == Cmn.DatabaseEngineEdition.SqlManagedInstance) || + (sp.TargetDatabaseEngineEdition == Cmn.DatabaseEngineEdition.SqlAzureArcManagedInstance)); ScriptAlterPropBool("AnsiNullDefault", "ANSI_NULL_DEFAULT", sp, query); ScriptAlterPropBool("AnsiNullsEnabled", "ANSI_NULLS", sp, query); @@ -6546,7 +6552,7 @@ private void ScriptDbOptionsProps(StringCollection query, ScriptingPreferences s if (this.IsSupportedProperty("ContainmentType", sp) && !targetEditionIsManagedServer) { - ContainmentType cType = this.GetPropValueOptional("ContainmentType", ContainmentType.None); + ContainmentType cType = this.GetPropValueOptional("ContainmentType", ContainmentType.None); if (cType != ContainmentType.None) { this.AddDefaultLanguageOption("DefaultFullTextLanguageName", "DefaultFullTextLanguageLcid", @@ -6737,7 +6743,8 @@ internal void ScriptAlterPropReadonly(StringCollection query, ScriptingPreferenc if (IsSupportedProperty(nameof(ReadOnly)) && IsSupportedProperty(nameof(ReadOnly), sp) && sp.TargetDatabaseEngineEdition != DatabaseEngineEdition.SqlDataWarehouse && - sp.TargetDatabaseEngineEdition != DatabaseEngineEdition.SqlManagedInstance) + sp.TargetDatabaseEngineEdition != DatabaseEngineEdition.SqlManagedInstance && + sp.TargetDatabaseEngineEdition != DatabaseEngineEdition.SqlAzureArcManagedInstance) { // Specify READONLY or READWRITE based on the readonlyMode passed in, ignoring alters for dirty-only, etc. ScriptAlterPropBool("ReadOnly", "", sp, query, readonlyMode ? "READ_ONLY" : "READ_WRITE"); @@ -6775,7 +6782,7 @@ void ScriptAlterContainmentDDL(ScriptingPreferences sp, StringCollection queries { if (this.IsSupportedProperty("ContainmentType", sp)) { - ContainmentType cType = this.GetPropValueOptional("ContainmentType", ContainmentType.None); + ContainmentType cType = this.GetPropValueOptional("ContainmentType", ContainmentType.None); switch (cType) { case ContainmentType.None: @@ -7071,5 +7078,31 @@ public void CleanupPersistentVersionStore() { ExecuteNonQuery($"exec sys.sp_persistent_version_cleanup {MakeSqlBraket(Name)}"); } + + /// + /// Populates the object's property bag from the current row of the DataReader + /// + /// + /// If true do not initialize the property if it has + /// been changed by the user + /// Index of the first column + /// Index of the last column. If -1 then go to the end. + internal override void AddObjectPropsFromDataReader(IDataReader reader, bool skipIfDirty, + int startColIdx, int endColIdx) + { + // We need the DatabaseEngineEdition for initializing the properties list for a Database, but this + // can cause problems on Azure servers since getting the EngineEdition requires logging into the + // database itself which is something we want to avoid for serverless databases or inaccessible + // databases. So to avoid that we prepopulate the edition by checking if it's DW beforehand (which + // doesn't require connecting to the database to retrieve) + if (m_edition == null && this.Parent.DatabaseEngineType == DatabaseEngineType.SqlAzureDatabase) + { + if (reader.GetSchemaTable().Rows.Cast().FirstOrDefault(r=> (string)r["ColumnName"] == "RealEngineEdition") != null) + { + this.m_edition = (DatabaseEngineEdition)reader["RealEngineEdition"]; + } + } + base.AddObjectPropsFromDataReader(reader, skipIfDirty, startColIdx, endColIdx); + } } } diff --git a/src/Microsoft/SqlServer/Management/Smo/ExternalFileFormatBase.cs b/src/Microsoft/SqlServer/Management/Smo/ExternalFileFormatBase.cs index d3f6908e..6b201347 100644 --- a/src/Microsoft/SqlServer/Management/Smo/ExternalFileFormatBase.cs +++ b/src/Microsoft/SqlServer/Management/Smo/ExternalFileFormatBase.cs @@ -164,7 +164,7 @@ internal override void ScriptCreate(StringCollection createQuery, ScriptingPrefe this.ThrowIfNotSupported(typeof(ExternalFileFormat), sp); /* CREATE EXTERNAL FILE FORMAT external_file_format_name WITH - * (FORMAT_TYPE = { DELIMITEDTEXT | RCFILE | ORC | PARQUET | JSON } + * (FORMAT_TYPE = { DELIMITEDTEXT | RCFILE | ORC | PARQUET | JSON | DELTA } * ,[SERDE_METHOD = 'Serialization/Deserialization method'] * ,[FORMAT_OPTIONS ( [ ,...n ] )] * ,[DATA_COMPRESSION = 'data_compression_method'] @@ -219,9 +219,6 @@ internal override void ScriptCreate(StringCollection createQuery, ScriptingPrefe sb.Append(Globals.LParen); sb.AppendFormat(SmoApplication.DefaultCulture, "FORMAT_TYPE = {0}", typeConverter.ConvertToInvariantString(externalFileFormatType)); - // check for conflicting properties - CheckConflictingProperties(sp); - // add any optional properties if they are set ProcessOptionalProperties(externalFileFormatType, sb, sp); @@ -257,122 +254,6 @@ private void AddPropertyToScript(string propertyValue, string sqlString, StringB fileFormatOptions.AppendFormat(SmoApplication.DefaultCulture, sqlString, propertyValue); } - /// - /// Checks for conflicting properties for the specified external file format type. - /// If a conflicting configuration is detected, throws an exception. - /// - private void CheckConflictingProperties(ScriptingPreferences sp) - { - const string FormatTypePropertyName = "FormatType"; - const string SerDeMethodPropertyName = "SerDeMethod"; - const string FieldTerminatorPropertyName = "FieldTerminator"; - const string StringDelimiterPropertyName = "StringDelimiter"; - const string DateFormatPropertyName = "DateFormat"; - const string UseTypeDefaultPropertyName = "UseTypeDefault"; - - // confirm that for each of the file format types, - // correct optional properties are specified, if any - // ensure that the format type property has a value - if (IsSupportedProperty(FormatTypePropertyName, sp)) - { - Property formatTypeProp = this.GetPropertyOptional(FormatTypePropertyName); - if (!formatTypeProp.IsNull) - { - Property prop = null; - - // if the format type is either delimited text, orc, parquet, or json then the serde method property is not supported - if ((ExternalFileFormatType)formatTypeProp.Value == ExternalFileFormatType.DelimitedText || - (ExternalFileFormatType)formatTypeProp.Value == ExternalFileFormatType.Orc || - (ExternalFileFormatType)formatTypeProp.Value == ExternalFileFormatType.Parquet || - (ExternalFileFormatType)formatTypeProp.Value == ExternalFileFormatType.JSON) - { - // check the serde method property - // if it is specified and not a default, throw an exception - if (IsSupportedProperty(SerDeMethodPropertyName, sp)) - { - prop = this.GetPropertyOptional(SerDeMethodPropertyName); - if (!prop.IsNull) - { - // if the property is set to something other than its default value, throw an exception - if (!IsPropertyDefaultValue(prop, (string)prop.Value, new List { null, string.Empty })) - { - throw new SmoException(string.Format(SmoApplication.DefaultCulture, ExceptionTemplates.ConflictingExternalFileFormatProperties, prop.Name, prop.Value.ToString(), formatTypeProp.Value.ToString())); - } - } - } - } - - // if the format type is rcfile, orc, parquet or json then the format options properties are not supported - if ((ExternalFileFormatType)formatTypeProp.Value == ExternalFileFormatType.RcFile || - (ExternalFileFormatType)formatTypeProp.Value == ExternalFileFormatType.Orc || - (ExternalFileFormatType)formatTypeProp.Value == ExternalFileFormatType.Parquet || - (ExternalFileFormatType)formatTypeProp.Value == ExternalFileFormatType.JSON) - { - // check the format options properties - // if any of them are specified, throw an exception - if (IsSupportedProperty(FieldTerminatorPropertyName, sp)) - { - prop = this.GetPropertyOptional(FieldTerminatorPropertyName); - if (!prop.IsNull) - { - if (!IsPropertyDefaultValue(prop, (string)prop.Value, new List { null, string.Empty })) - { - throw new SmoException(string.Format(SmoApplication.DefaultCulture, ExceptionTemplates.ConflictingExternalFileFormatProperties, prop.Name, prop.Value.ToString(), formatTypeProp.Value.ToString())); - } - } - } - - if (IsSupportedProperty(StringDelimiterPropertyName, sp)) - { - prop = this.GetPropertyOptional(StringDelimiterPropertyName); - if (!prop.IsNull) - { - if (!IsPropertyDefaultValue(prop, (string)prop.Value, new List { null, string.Empty })) - { - throw new SmoException(string.Format(SmoApplication.DefaultCulture, ExceptionTemplates.ConflictingExternalFileFormatProperties, prop.Name, prop.Value.ToString(), formatTypeProp.Value.ToString())); - } - } - } - - if (IsSupportedProperty(DateFormatPropertyName, sp)) - { - prop = this.GetPropertyOptional(DateFormatPropertyName); - if (!prop.IsNull) - { - if (!IsPropertyDefaultValue(prop, (string)prop.Value, new List { null, string.Empty })) - { - throw new SmoException(string.Format(SmoApplication.DefaultCulture, ExceptionTemplates.ConflictingExternalFileFormatProperties, prop.Name, prop.Value.ToString(), formatTypeProp.Value.ToString())); - } - } - } - - if (IsSupportedProperty(UseTypeDefaultPropertyName, sp)) - { - prop = this.GetPropertyOptional(UseTypeDefaultPropertyName); - if (!prop.IsNull) - { - if (!IsPropertyDefaultValue(prop, (bool)prop.Value, new List { false })) - { - throw new SmoException(string.Format(SmoApplication.DefaultCulture, ExceptionTemplates.ConflictingExternalFileFormatProperties, prop.Name, prop.Value.ToString(), formatTypeProp.Value.ToString())); - } - } - } - - if (IsSupportedProperty(FirstRowName, sp)) - { - prop = this.GetPropertyOptional(FirstRowName); - if (!prop.IsNull) - { - if (!IsPropertyDefaultValue(prop, (int)prop.Value, new List { 0 })) - { - throw new SmoException(string.Format(SmoApplication.DefaultCulture, ExceptionTemplates.ConflictingExternalFileFormatProperties, prop.Name, prop.Value.ToString(), formatTypeProp.Value.ToString())); - } - } - } - } - } - } - } /// /// Check the specified property if it has the default value. @@ -409,42 +290,19 @@ private bool IsPropertyDefaultValue(Property prop, T value, List defaultVa /// The scripting preferences. private void ProcessOptionalProperties(ExternalFileFormatType externalFileFormatType, StringBuilder script, ScriptingPreferences sp) { - // check the optional properties are supported by the specified format type - switch (externalFileFormatType) - { - case ExternalFileFormatType.DelimitedText: - ValidateDelimitedTextProperties(script, sp); - break; - case ExternalFileFormatType.JSON: - case ExternalFileFormatType.Orc: - case ExternalFileFormatType.Parquet: - ValidateOrcOrParquetProperties(script, sp); - break; - case ExternalFileFormatType.RcFile: - ValidateRcFileProperties(script, sp); - break; - default: - // if the format type set to any other value throw an exception - throw new WrongPropertyValueException(ExceptionTemplates.UnknownEnumeration(externalFileFormatType.ToString())); - } - } - - /// - /// Validates optional properties for the DelimtedText file format - /// and adds them to the T-SQL script. - /// - /// The external file format T-SQL script. - /// The scripting preferences. - private void ValidateDelimitedTextProperties(StringBuilder script, ScriptingPreferences sp) - { - const string UseTypeDefaultPropertyName = "UseTypeDefault"; - // check for the DelimitedText supported optional properties - FormatOptions and DataCompression // check for any format options optinal parameters - FieldTerminator, StringDelimiter, DateFormat and UseTypeDefault // if any are found, add them to the T-SQL script StringBuilder formatOptions = new StringBuilder(Globals.INIT_BUFFER_SIZE); List defaultValues = new List { null, string.Empty }; + const string UseTypeDefaultPropertyName = "UseTypeDefault"; + + + // check if the serde method property was set + // if yes, add it to the script + ValidateOptionalProperty("SerDeMethod", "SERDE_METHOD = {0}", defaultValues, script, sp); + // validate and process the field terminator file format option ValidateOptionalProperty("FieldTerminator", "FIELD_TERMINATOR = {0}", defaultValues, formatOptions, sp); @@ -455,23 +313,31 @@ private void ValidateDelimitedTextProperties(StringBuilder script, ScriptingPref ValidateOptionalProperty("DateFormat", "DATE_FORMAT = {0}", defaultValues, formatOptions, sp); // validate and process the first row optional file format property - ValidateOptionalProperty(FirstRowName, "FIRST_ROW = {0}", new List { 1 }, formatOptions, sp, quotePropertyValue: false); + // for delimited text default value is 1, and for the rest the default value is 0. + if (externalFileFormatType == ExternalFileFormatType.DelimitedText) + { + ValidateOptionalProperty(FirstRowName, "FIRST_ROW = {0}", new List { 1 }, formatOptions, sp, quotePropertyValue: false); + } else + { + ValidateOptionalProperty(FirstRowName, "FIRST_ROW = {0}", new List { 0 }, formatOptions, sp, quotePropertyValue: false); + } // validate and process the use type default file format option if (IsSupportedProperty(UseTypeDefaultPropertyName, sp)) { - if (!this.GetPropertyOptional(UseTypeDefaultPropertyName).IsNull) + var prop = this.GetPropertyOptional(UseTypeDefaultPropertyName); + // property is ignored if it's null or has default value + if(!prop.IsNull && (externalFileFormatType == ExternalFileFormatType.DelimitedText || !IsPropertyDefaultValue(prop, (bool)prop.Value, new List { false }))) { bool externalFileFormatUseTypeDefault = (bool)this.GetPropValueOptional(UseTypeDefaultPropertyName); - if (formatOptions.Length > 0) { formatOptions.Append(", "); } - - formatOptions.AppendFormat(SmoApplication.DefaultCulture, "USE_TYPE_DEFAULT = {0}", externalFileFormatUseTypeDefault); + formatOptions.AppendFormat(SmoApplication.DefaultCulture, "USE_TYPE_DEFAULT = {0}", externalFileFormatUseTypeDefault); } } + // if any format options were specified, add the FORMAT_OPTIONS and enclose them in the parenthesis string fileFormatOptions = formatOptions.ToString(); @@ -515,12 +381,12 @@ private void ValidateOptionalProperty(string propertyName, string sqlString, } /// - /// Validates optional properties for the Orc or Parquet file format + /// Validates optional properties for the JSON, Orc, Parquet or Delta file format /// and adds them to the T-SQL script. /// /// The external file format T-SQL script. /// The scripting preferences. - private void ValidateOrcOrParquetProperties(StringBuilder script, ScriptingPreferences sp) + private void ValidateOrcParquetJsonOrDeltaProperties(StringBuilder script, ScriptingPreferences sp) { List defaultValues = new List { null, string.Empty }; diff --git a/src/Microsoft/SqlServer/Management/Smo/Microsoft.SqlServer.Smo.csproj b/src/Microsoft/SqlServer/Management/Smo/Microsoft.SqlServer.Smo.csproj index 911e5b39..fbeddbcc 100644 --- a/src/Microsoft/SqlServer/Management/Smo/Microsoft.SqlServer.Smo.csproj +++ b/src/Microsoft/SqlServer/Management/Smo/Microsoft.SqlServer.Smo.csproj @@ -669,4 +669,3 @@ - diff --git a/src/Microsoft/SqlServer/Management/Smo/SqlSmoObject.cs b/src/Microsoft/SqlServer/Management/Smo/SqlSmoObject.cs index 77a78611..fff27657 100644 --- a/src/Microsoft/SqlServer/Management/Smo/SqlSmoObject.cs +++ b/src/Microsoft/SqlServer/Management/Smo/SqlSmoObject.cs @@ -1764,7 +1764,7 @@ internal void AddObjectPropsFromDataReader(System.Data.IDataReader reader, bool /// been changed by the user /// Index of the first column /// Index of the last column. If -1 then go to the end. - internal void AddObjectPropsFromDataReader(System.Data.IDataReader reader, bool skipIfDirty, + internal virtual void AddObjectPropsFromDataReader(System.Data.IDataReader reader, bool skipIfDirty, int startColIdx, int endColIdx) { var schemaTable = reader.GetSchemaTable(); @@ -4263,6 +4263,11 @@ public IEnumerable GetDisabledProperties(ScriptingPreferences sp = null) yield return nameof(Index.IsSpatialIndex); yield return nameof(Server.Configuration.ContainmentEnabled); } + if (((sp != null && sp.TargetDatabaseEngineEdition != DatabaseEngineEdition.SqlOnDemand) || this.DatabaseEngineEdition != DatabaseEngineEdition.SqlOnDemand) && + ((sp != null && sp.TargetDatabaseEngineEdition != DatabaseEngineEdition.SqlDataWarehouse) || this.DatabaseEngineEdition != DatabaseEngineEdition.SqlDataWarehouse)) + { + yield return nameof(ExternalFileFormat.FirstRow); + } if ((sp != null && sp.TargetDatabaseEngineEdition == DatabaseEngineEdition.SqlOnDemand) || this.DatabaseEngineEdition == DatabaseEngineEdition.SqlOnDemand) { yield return nameof(Database.AutoClose); diff --git a/src/Microsoft/SqlServer/Management/Smo/enumerations.cs b/src/Microsoft/SqlServer/Management/Smo/enumerations.cs index 737dc481..5b8a4abb 100644 --- a/src/Microsoft/SqlServer/Management/Smo/enumerations.cs +++ b/src/Microsoft/SqlServer/Management/Smo/enumerations.cs @@ -419,12 +419,14 @@ make sure to update both enums with any changes SqlManagedInstance = 0x000008, ///The server is a SQL Edge Instance SqlDatabaseEdge = 0x000009, + ///The server is an Azure Arc Managed SQL Instance + SqlAzureArcManagedInstance = 0x00000A, ///The server is SQL SqlOnDemand SqlOnDemand = 0x00000B, /* * NOTE: If you're adding new value here, * please update as well ScriptDatabaseEngineEdition enum - * in Sql\ssms\extensions\synthesis\NextGenDPW\SqlScriptPublishModel\SqlScriptOptions.cs + * in src\Microsoft\SqlServer\Management\SqlScriptPublish\SqlScriptOptions.cs */ } diff --git a/src/Microsoft/SqlServer/Management/Smo/serverbase.cs b/src/Microsoft/SqlServer/Management/Smo/serverbase.cs index ab090cb4..b55dab29 100644 --- a/src/Microsoft/SqlServer/Management/Smo/serverbase.cs +++ b/src/Microsoft/SqlServer/Management/Smo/serverbase.cs @@ -3089,8 +3089,6 @@ private StringCollection CreateInitFieldsColl(Type typeObject) // but we need them at runtime when we initialize the object, and we'd // rather have them in the collection for performance consideration - // TODO: FIX_IN_KATMAI: Why does Server need to know about all of its children? Delegate the call to the child. - if (typeObject.IsSubclassOf(typeof(ScriptSchemaObjectBase))) { fields.Add("Schema"); @@ -3131,6 +3129,17 @@ private StringCollection CreateInitFieldsColl(Type typeObject) fields.Add("CategoryID"); fields.Add("JobID"); } + else if (typeObject == typeof(Database)) + { + fields.Add(nameof(Database.Name)); + // If we are connected to an OnDemand or DataWarehouse instance through the ServerConnection + // the Database can only be the same edition as the ServerConnection. + // If we are connected to logical master then Database can have a different edition, so + // add RealEngineEdition to the query so we get the edition from sys.database_service_objectives + if (DatabaseEngineEdition == DatabaseEngineEdition.SqlDatabase) { + fields.Add("RealEngineEdition"); + } + } else if (typeObject.IsSubclassOf(typeof(NamedSmoObject))) { fields.Add("Name"); diff --git a/src/Microsoft/SqlServer/Management/Smo/tablebase.cs b/src/Microsoft/SqlServer/Management/Smo/tablebase.cs index 2434525f..06eb9648 100644 --- a/src/Microsoft/SqlServer/Management/Smo/tablebase.cs +++ b/src/Microsoft/SqlServer/Management/Smo/tablebase.cs @@ -642,7 +642,6 @@ internal override void ScriptCreate(StringCollection queries, ScriptingPreferenc } bool fAnsiNullsExists = false; - bool fQuotedIdentifierExists = false; bool ansiPaddingStatus = false; if (Cmn.DatabaseEngineType.SqlAzureDatabase != this.DatabaseEngineType) @@ -676,14 +675,10 @@ internal override void ScriptCreate(StringCollection queries, ScriptingPreferenc sb.Length = 0; } - fQuotedIdentifierExists = (null != Properties.Get("QuotedIdentifierStatus").Value); - if (fQuotedIdentifierExists) - { - sb.AppendFormat(SmoApplication.DefaultCulture, Scripts.SET_QUOTED_IDENTIFIER, - (bool)Properties["QuotedIdentifierStatus"].Value ? Globals.On : Globals.Off); - scqueries.Add(sb.ToString()); - sb.Length = 0; - } + // QUOTED_IDENTIFIER in Tables metadata is always ON + sb.AppendFormat(SmoApplication.DefaultCulture, Scripts.SET_QUOTED_IDENTIFIER, Globals.On); + scqueries.Add(sb.ToString()); + sb.Length = 0; } // set the ANSI_PADDING only if the table creation script @@ -2301,22 +2296,12 @@ private void ProcessSqlDwTableProperties(StringBuilder script, ScriptingPreferen { case DwTableDistributionType.Hash: - // get the distribution column name - var distributionColumnNameList = new List(); - - foreach (Column col in this.Columns) - { - if (col.GetPropValueOptional(IsDistributedColumnPropertyName, false)) - { - distributionColumnNameList.Add(col.GetPropValueOptional(DistributionColumnNamePropertyName, string.Empty)); - break; - } - } - - string distributionColumnNames = string.Join(",", distributionColumnNameList.Select(x => MakeSqlBraket(x))); - string distributionWithDistributionColName = string.Format("{0} ( {1} )", - typeConverter.ConvertToInvariantString(distribution), - distributionColumnNames); + // get the distribution column names + var distributionColumnNames = string.Join(",", Columns.Cast() + .Where(col => col.GetPropValueOptional(IsDistributedColumnPropertyName, false)) + .Select(col => MakeSqlBraket(col.GetPropValueOptional(DistributionColumnNamePropertyName, string.Empty)))); + + var distributionWithDistributionColName = $"{typeConverter.ConvertToInvariantString(distribution)} ( {distributionColumnNames} )"; this.AddPropertyToScript(distributionWithDistributionColName, "DISTRIBUTION = {0}", script); break; @@ -4546,7 +4531,6 @@ internal static string[] GetScriptFields(Type parentType, "RejectSampleValue", "RejectType", "RejectValue", - "QuotedIdentifierStatus", "RemoteObjectName", "RemoteSchemaName", "ShardingColumnName", diff --git a/src/Microsoft/SqlServer/Management/SqlEnum/SqlConditionedStatement.cs b/src/Microsoft/SqlServer/Management/SqlEnum/SqlConditionedStatement.cs index 6c9232a5..1934b848 100644 --- a/src/Microsoft/SqlServer/Management/SqlEnum/SqlConditionedStatement.cs +++ b/src/Microsoft/SqlServer/Management/SqlEnum/SqlConditionedStatement.cs @@ -135,4 +135,29 @@ public override void AddHit(string field, SqlObjectBase obj, StatementBuilder sb sb.AddCondition(this.GetLocalSql(obj)); } } + + internal class SqlConditionedStatementWhereClause : SqlConditionedStatement + { + public SqlConditionedStatementWhereClause(XmlReadSpecialQuery xrcs) : base(xrcs) + { + } + + public static void AddAll(ConditionedSqlList list, XmlReadSpecialQuery xrcs) + { + if (null != xrcs) + { + do + { + list.Add(new SqlConditionedStatementWhereClause(xrcs)); + } + while (xrcs.Next()); + } + } + /// + ///add hit for field + public override void AddHit(string field, SqlObjectBase obj, StatementBuilder sb) + { + sb.AddWhere(this.GetLocalSql(obj)); + } + } } diff --git a/src/Microsoft/SqlServer/Management/SqlEnum/SqlObject.cs b/src/Microsoft/SqlServer/Management/SqlEnum/SqlObject.cs index 0dc56748..33840ea3 100644 --- a/src/Microsoft/SqlServer/Management/SqlEnum/SqlObject.cs +++ b/src/Microsoft/SqlServer/Management/SqlEnum/SqlObject.cs @@ -176,10 +176,20 @@ internal virtual void Load(XmlReadDoc xrd, Assembly assembly, StringCollection r xrobr.Close(); } - XmlReadSpecialQuery xrsq = xrs.SpecialQuery; + var xrsq = xrs.SpecialQuery; if (null != xrsq) { - this.AddSpecialQuery(xrsq.Database, xrsq.Query); + // Preserve original behavior - database-specific where clause + var database = xrsq.Database; + if (!string.IsNullOrEmpty(database)) + { + this.AddSpecialQuery(xrsq.Database, xrsq.Sql); + } + else + // Where clause based on Fields + { + SqlConditionedStatementWhereClause.AddAll(ConditionedSqlList, xrsq); + } this.AddQueryHint(xrsq.Hint); xrsq.Close(); } diff --git a/src/Microsoft/SqlServer/Management/SqlEnum/XmlRead.CS b/src/Microsoft/SqlServer/Management/SqlEnum/XmlRead.CS index 9890354b..e63fa432 100644 --- a/src/Microsoft/SqlServer/Management/SqlEnum/XmlRead.CS +++ b/src/Microsoft/SqlServer/Management/SqlEnum/XmlRead.CS @@ -18,6 +18,9 @@ namespace Microsoft.SqlServer.Management.Smo #endif using Microsoft.SqlServer.Management.Sdk.Sfc; using Microsoft.SqlServer.Management.Smo.SqlEnum; + using System.Collections.Generic; + using System.Text; + /// ///base class for parsing a xml configuration file [ComVisible(false)] @@ -305,23 +308,23 @@ namespace Microsoft.SqlServer.Management.Smo ///read a 'fields' entry ( list of fields separated by '#' ) static protected StringCollection GetFields(string fields) { - String fieldName = String.Empty; - StringCollection col = new StringCollection(); + var fieldName = new StringBuilder(); + var col = new StringCollection(); if( null != fields ) { - foreach(char c in fields) + foreach(var c in fields) { if( '#' == c ) { if( 0 < fieldName.Length ) { - col.Add(fieldName); + col.Add(fieldName.ToString()); } - fieldName = String.Empty; + fieldName = new StringBuilder(); continue; } - fieldName += c; + fieldName.Append(c); } } return col; @@ -329,7 +332,7 @@ namespace Microsoft.SqlServer.Management.Smo /// ///reads the text of the current element if available - ///it assumes the curent element is empty or has inside it either a text node or + ///it assumes the current element is empty or has inside it either a text node or ///an element node. probably a link_multiple. protected string GetTextOfElement() { @@ -349,7 +352,7 @@ namespace Microsoft.SqlServer.Management.Smo //we found text if( XmlNodeType.Text == this.Reader.NodeType ) { - //so get it's value + //so get its value return this.Reader.Value; } } @@ -1759,7 +1762,7 @@ namespace Microsoft.SqlServer.Management.Smo /// ///class to read a special query [ComVisible(false)] - internal class XmlReadSpecialQuery : XmlRead + internal class XmlReadSpecialQuery : XmlReadConditionedStatement { /// ///initialize with reader @@ -1774,12 +1777,6 @@ namespace Microsoft.SqlServer.Management.Smo get { return this.Reader["database"]; } } - /// - ///get attribute - query - public String Query - { - get { return this.Reader["query"]; } - } /// /// Used to get attribute 'query hint' @@ -1793,5 +1790,10 @@ namespace Microsoft.SqlServer.Management.Smo { get { return this.Reader["hint"]; } } + + public override bool Next() + { + return Next("special_query"); + } } } diff --git a/src/Microsoft/SqlServer/Management/SqlEnum/enumstructs.cs b/src/Microsoft/SqlServer/Management/SqlEnum/enumstructs.cs index dcbab44c..e28edfeb 100644 --- a/src/Microsoft/SqlServer/Management/SqlEnum/enumstructs.cs +++ b/src/Microsoft/SqlServer/Management/SqlEnum/enumstructs.cs @@ -339,12 +339,18 @@ public enum ExternalFileFormatType /// [TsqlSyntaxString("PARQUET")] Parquet = 3, - + /// /// JSON file format. /// [TsqlSyntaxString("JSON")] - JSON = 4 + JSON = 4, + + /// + /// DELTA file format. + /// + [TsqlSyntaxString("DELTA")] + Delta = 5 } /// @@ -3123,6 +3129,11 @@ public enum AuditActionType /// [TsqlSyntaxString("SENSITIVE_BATCH_COMPLETED_GROUP")] SensitiveBatchCompletedGroup, + /// + /// EXTGOV_OPERATION_GROUP + /// + [TsqlSyntaxString("EXTGOV_OPERATION_GROUP")] + ExternalGovernanceOperationGroup, } /// diff --git a/src/Microsoft/SqlServer/Management/SqlEnum/permenum.cs b/src/Microsoft/SqlServer/Management/SqlEnum/permenum.cs index da275dcb..cafd8cca 100644 --- a/src/Microsoft/SqlServer/Management/SqlEnum/permenum.cs +++ b/src/Microsoft/SqlServer/Management/SqlEnum/permenum.cs @@ -297,6 +297,50 @@ public enum DatabasePermissionSetValue [PermissionType("EAEE")] [PermissionName("EXECUTE ANY EXTERNAL ENDPOINT")] ExecuteAnyExternalEndpoint = 93, + [PermissionType("CRDS")] + [PermissionName("CREATE ANY DATABASE EVENT SESSION")] + CreateAnyDatabaseEventSession = 94, + [PermissionType("DRDS")] + [PermissionName("DROP ANY DATABASE EVENT SESSION")] + DropAnyDatabaseEventSession = 95, + [PermissionType("LDSO")] + [PermissionName("ALTER ANY DATABASE EVENT SESSION OPTION")] + AlterAnyDatabaseEventSessionOption = 96, + [PermissionType("LDAE")] + [PermissionName("ALTER ANY DATABASE EVENT SESSION ADD EVENT")] + AlterAnyDatabaseEventSessionAddEvent = 97, + [PermissionType("LDDE")] + [PermissionName("ALTER ANY DATABASE EVENT SESSION DROP EVENT")] + AlterAnyDatabaseEventSessionDropEvent = 98, + + [PermissionType("EDES")] + [PermissionName("ALTER ANY DATABASE EVENT SESSION ENABLE")] + AlterAnyDatabaseEventSessionEnable = 99, + + [PermissionType("DDES")] + [PermissionName("ALTER ANY DATABASE EVENT SESSION DISABLE")] + AlterAnyDatabaseEventSessionDisable = 100, + + [PermissionType("LDAT")] + [PermissionName("ALTER ANY DATABASE EVENT SESSION ADD TARGET")] + AlterAnyDatabaseEventSessionAddTarget = 101, + + [PermissionType("LDDT")] + [PermissionName("ALTER ANY DATABASE EVENT SESSION DROP TARGET")] + AlterAnyDatabaseEventSessionDropTarget = 102, + + [PermissionType("VWP")] + [PermissionName("VIEW PERFORMANCE DEFINITION")] + ViewPerformanceDefinition = 103, + + [PermissionType("VDSA")] + [PermissionName("VIEW DATABASE SECURITY AUDIT")] + ViewDatabaseSecurityAudit = 104, + + [PermissionType("ALC")] + [PermissionName("ALTER LEDGER CONFIGURATION")] + AlterLedgerConfiguration = 105, + } ///enum containing all possible ObjectPermissions @@ -485,7 +529,56 @@ public enum ServerPermissionSetValue ViewServerPerformanceState = 39, [PermissionType("VACD")] [PermissionName("VIEW ANY CRYPTOGRAPHICALLY SECURED DEFINITION")] - ViewAnyCryptographicallySecuredDefinition = 40 + ViewAnyCryptographicallySecuredDefinition = 40, + + [PermissionType("VAP")] + [PermissionName("VIEW ANY PERFORMANCE DEFINITION")] + ViewAnyPerformanceDefinition = 41, + + [PermissionType("CRES")] + [PermissionName("CREATE ANY EVENT SESSION")] + CreateAnyEventSession = 42, + + [PermissionType("DRES")] + [PermissionName("DROP ANY EVENT SESSION")] + DropAnyEventSession = 43, + + [PermissionType("LESO")] + [PermissionName("ALTER ANY EVENT SESSION OPTION")] + AlterAnyEventSessionOption = 44, + + [PermissionType("LSAE")] + [PermissionName("ALTER ANY EVENT SESSION ADD EVENT")] + AlterAnyEventSessionAddEvent = 45, + + [PermissionType("LSDE")] + [PermissionName("ALTER ANY EVENT SESSION DROP EVENT")] + AlterAnyEventSessionDropEvent = 46, + + [PermissionType("EES")] + [PermissionName("ALTER ANY EVENT SESSION ENABLE")] + AlterAnyEventSessionEnable = 47, + + [PermissionType("DES")] + [PermissionName("ALTER ANY EVENT SESSION DISABLE")] + AlterAnyEventSessionDisable = 48, + + [PermissionType("LSAT")] + [PermissionName("ALTER ANY EVENT SESSION ADD TARGET")] + AlterAnyEventSessionAddTarget = 49, + + [PermissionType("LSDT")] + [PermissionName("ALTER ANY EVENT SESSION DROP TARGET")] + AlterAnyEventSessionDropTarget = 50, + + [PermissionType("VEL")] + [PermissionName("VIEW ANY ERROR LOG")] + ViewAnyErrorLog = 51, + + [PermissionType("VSSA")] + [PermissionName("VIEW SERVER SECURITY AUDIT")] + ViewServerSecurityAudit = 52, + } ///encapsulates functions that translate from sql codes into enum used to represent the permissions diff --git a/src/Microsoft/SqlServer/Management/SqlEnum/xml/Database.xml b/src/Microsoft/SqlServer/Management/SqlEnum/xml/Database.xml index 4656fd34..ac7dbc4c 100644 --- a/src/Microsoft/SqlServer/Management/SqlEnum/xml/Database.xml +++ b/src/Microsoft/SqlServer/Management/SqlEnum/xml/Database.xml @@ -1,5 +1,5 @@ - + @@ -29,7 +29,7 @@ - + @@ -47,7 +47,7 @@ - dso.database_id = dtb.database_id + dso.database_id = dtb.database_id @@ -88,24 +88,26 @@ - - - create table #dso (database_id int primary key, isDw bit) + + + create table #dso (database_id int primary key, azureEdition sysname null, engineEdition int) + if serverproperty('EngineEdition') = 11 + BEGIN + insert into #dso select database_id, NULL, 11 from sys.databases + END + ELSE BEGIN TRY - insert into #dso select database_id, CAST( - case - when Edition = 'DataWarehouse' then cast(1 as bit) - else cast(0 as bit) - end - AS bit) from sys.database_service_objectives + exec sp_executesql N' + insert into #dso select database_id, edition, + case when edition = ''DataWarehouse'' then 6 else 5 end + from sys.database_service_objectives' END TRY BEGIN CATCH - insert into #dso select database_id, cast(0 as bit) from sys.databases + insert into #dso select database_id, N'', case when [name] = 'master' then 5 else 0 end from sys.databases END CATCH - - drop table #dso - @@ -154,6 +156,11 @@ + + + drop table #dso + + - + @@ -224,7 +231,10 @@ class_name='Microsoft.SqlServer.Management.Smo.PostProcessDatabaseInsideAttribs' triggered_fields='#DatabaseName7#'/> - + + + dso.engineEdition != 0 + @@ -251,7 +261,7 @@ dtb.status & <msparam>4194304</msparam> - + case when dtb.name in ('master') then 1 else dtb.is_distributor end (select schema_name()) @@ -263,10 +273,14 @@ ( case dtb.is_read_only when 1 then 0 else 1 end) + - ISNULL(dso.isDw, 0) + case when isnull(dso.engineEdition, 0) = 6 then 1 else 0 end + ISNULL(dso.azureEdition, N'') + case when dtb.name = 'master' then ISNULL(dso.engineEdition, 5) else ISNULL(dso.engineEdition, 0) end - + + ISNULL((select top 1 ftc.name from sys.fulltext_catalogs as ftc where ftc.is_default=1),N'') @@ -304,7 +318,7 @@ 0 - + dtb.is_ansi_null_default_on dtb.is_ansi_nulls_on @@ -333,7 +347,7 @@ dtb.compatibility_level dtb.is_read_committed_snapshot_on - + isnull(dtb.source_database_id, 0) (select count(1) from sys.databases dtbmir where dtbmir.source_database_id = dtb.database_id) @@ -439,7 +453,7 @@ - dtb.is_ledger_on + ISNULL(dtb.is_ledger_on, 0) @@ -531,8 +545,8 @@ - - DATABASEPROPERTYEX(dtb.name, 'Edition') + + CONVERT(float, DATABASEPROPERTYEX(dtb.Name, 'MaxSizeInBytes')) / 1024.0 / 1024.0 @@ -540,12 +554,13 @@ (CASE WHEN 0 > CONVERT(float, DATABASEPROPERTYEX(dtb.Name, 'MaxSizeInBytes')) THEN 0 ELSE 1 END) - + (SELECT IIF(databasepropertyex(dtb.name,'Edition') = 'Hyperscale', 0, (SELECT ((CAST(DATABASEPROPERTYEX(dtb.Name, 'MaxSizeInBytes') AS float) / 1024.0) - (SUM(reserved_page_count)*8.0 )) / 1024.0 FROM sys.dm_db_partition_stats))) 0 + N'DataWarehouse' @@ -556,7 +571,7 @@ CHARINDEX(N'_CS_', CAST(DATABASEPROPERTYEX(dtb.name, 'Collation') AS nvarchar(255))) - + case when CHARINDEX(N'_CS_', dtb.collation_name) > 0 then 1 when CHARINDEX(N'_BIN', dtb.collation_name) > 0 then 1 @@ -622,7 +637,7 @@ - + 0 0 diff --git a/src/Microsoft/SqlServer/Management/SqlEnum/xml/ExternalFileFormat.xml b/src/Microsoft/SqlServer/Management/SqlEnum/xml/ExternalFileFormat.xml index f62a9aa2..ff24944b 100644 --- a/src/Microsoft/SqlServer/Management/SqlEnum/xml/ExternalFileFormat.xml +++ b/src/Microsoft/SqlServer/Management/SqlEnum/xml/ExternalFileFormat.xml @@ -9,7 +9,7 @@ eff.name eff.file_format_id - CASE eff.format_type when 'DELIMITEDTEXT' then 0 when 'RCFILE' then 1 when 'ORC' then 2 when 'PARQUET' then 3 when 'JSON' then 4 END + CASE eff.format_type when 'DELIMITEDTEXT' then 0 when 'RCFILE' then 1 when 'ORC' then 2 when 'PARQUET' then 3 when 'JSON' then 4 when 'DELTA' then 5 END ISNULL(eff.field_terminator,N'') ISNULL(eff.string_delimiter,N'') @@ -19,7 +19,7 @@ ISNULL(eff.row_terminator,N'') ISNULL(eff.encoding,N'') ISNULL(eff.data_compression,N'') - + ISNULL(eff.first_row, 0) diff --git a/src/Microsoft/SqlServer/Management/SqlEnum/xml/README.md b/src/Microsoft/SqlServer/Management/SqlEnum/xml/README.md index 84b91693..31c17222 100644 --- a/src/Microsoft/SqlServer/Management/SqlEnum/xml/README.md +++ b/src/Microsoft/SqlServer/Management/SqlEnum/xml/README.md @@ -128,3 +128,8 @@ Post Processing is for doing additional calculations on the returned data. This - class_name: The name of the class that handles the post-processing. Should extend [PostProcess](../PostProcess.cs) - fields: The list of fields which will cause the post-processing to happen - triggered_fields: The list of fields that are needed to compute the value for the field requested by the user + +### special_query + +This tag has 2 specialized uses and one standard behavior. Originally it was a special case for [table.xml](table.xml), to add a filter to hide temp tables when enumerating tables in tempdb, and to enable adding a query hint for optimizing the overall tables query. +Now it also acts as similarly to `post_process` by being a general purpose conditioned sql tag with a `fields` attribute and a body which is added to the `WHERE` clause of the query with an `AND` condition. diff --git a/src/Microsoft/SqlServer/Management/SqlEnum/xml/inc_server.xml b/src/Microsoft/SqlServer/Management/SqlEnum/xml/inc_server.xml index 0da6a40f..d5a7bd96 100644 --- a/src/Microsoft/SqlServer/Management/SqlEnum/xml/inc_server.xml +++ b/src/Microsoft/SqlServer/Management/SqlEnum/xml/inc_server.xml @@ -385,7 +385,7 @@ FROM sys.dm_hadr_cluster END TRY BEGIN CATCH - IF(ERROR_NUMBER() NOT IN (297,300, 15562)) + IF(ERROR_NUMBER() NOT IN (297,300, 15562, 371)) BEGIN THROW END diff --git a/src/Microsoft/SqlServer/Management/SqlEnum/xml/table.xml b/src/Microsoft/SqlServer/Management/SqlEnum/xml/table.xml index 55985fcc..aaab6700 100644 --- a/src/Microsoft/SqlServer/Management/SqlEnum/xml/table.xml +++ b/src/Microsoft/SqlServer/Management/SqlEnum/xml/table.xml @@ -184,7 +184,7 @@ triggered_fields='#DatabaseName#SchemaName#TableName#'/> - + tbl.name not like '#%' @@ -265,7 +265,7 @@ ISNULL((select top 1 1 from sys.all_columns as clmns join sys.types as usrt on usrt.user_type_id = clmns.user_type_id where clmns.object_id = tbl.object_id and usrt.name = N'xml'), 0) ISNULL((select top 1 1 from sys.all_columns as clmns join sys.types as usrt on usrt.user_type_id = clmns.user_type_id where clmns.object_id = tbl.object_id and usrt.name in (N'geometry', N'geography')), 0) tbl.uses_ansi_nulls - ISNULL(OBJECTPROPERTY(tbl.object_id,N'IsQuotedIdentOn'),0) + ISNULL(OBJECTPROPERTY(tbl.object_id,N'IsQuotedIdentOn'),1) 0 diff --git a/src/Microsoft/SqlServer/Management/SqlScriptPublish/SqlScriptOptions.cs b/src/Microsoft/SqlServer/Management/SqlScriptPublish/SqlScriptOptions.cs index c5cdf415..9d7dcf94 100644 --- a/src/Microsoft/SqlServer/Management/SqlScriptPublish/SqlScriptOptions.cs +++ b/src/Microsoft/SqlServer/Management/SqlScriptPublish/SqlScriptOptions.cs @@ -78,7 +78,9 @@ public enum ScriptDatabaseEngineEdition [DisplayNameKey("SqlServerOnDemandEdition")] SqlServerOnDemandEdition, [DisplayNameKey("SqlDatabaseEdgeEdition")] - SqlDatabaseEdgeEdition + SqlDatabaseEdgeEdition, + [DisplayNameKey("SqlAzureArcManagedInstanceEdition")] + SqlAzureArcManagedInstanceEdition, /* * NOTE: If you're adding new value here, @@ -328,6 +330,7 @@ public ICollection ConfigureVisibleEnumFields(ITypeDescriptorContext context, Ar values.Remove(ScriptDatabaseEngineEdition.SqlDatabaseEdgeEdition); values.Remove(ScriptDatabaseEngineEdition.SqlAzureDatabaseEdition); values.Remove(ScriptDatabaseEngineEdition.SqlServerOnDemandEdition); + values.Remove(ScriptDatabaseEngineEdition.SqlAzureArcManagedInstanceEdition); } else { @@ -340,6 +343,7 @@ public ICollection ConfigureVisibleEnumFields(ITypeDescriptorContext context, Ar values.Remove(ScriptDatabaseEngineEdition.SqlServerManagedInstanceEdition); values.Remove(ScriptDatabaseEngineEdition.SqlDatabaseEdgeEdition); values.Remove(ScriptDatabaseEngineEdition.SqlDatawarehouseEdition); + values.Remove(ScriptDatabaseEngineEdition.SqlAzureArcManagedInstanceEdition); } } else diff --git a/src/Microsoft/SqlServer/Management/SqlScriptPublish/SqlScriptOptionsSR.strings b/src/Microsoft/SqlServer/Management/SqlScriptPublish/SqlScriptOptionsSR.strings index 436edfc0..2ceee6f1 100644 --- a/src/Microsoft/SqlServer/Management/SqlScriptPublish/SqlScriptOptionsSR.strings +++ b/src/Microsoft/SqlServer/Management/SqlScriptPublish/SqlScriptOptionsSR.strings @@ -235,6 +235,7 @@ StretchEdition = Microsoft SQL Server Stretch Database Edition SqlServerManagedInstanceEdition = Microsoft Azure SQL Database Managed Instance Edition SqlServerOnDemandEdition = Microsoft Azure SQL OnDemand Edition SqlDatabaseEdgeEdition = Microsoft Azure SQL Edge Edition +SqlAzureArcManagedInstanceEdition = Microsoft Azure Arc SQL Managed Instance Edition IncludeUnsupportedStatements = Include unsupported statements IncludeUnsupportedStatementsDescription = Include statements in the script that are not supported on the specified SQL Server database engine type. diff --git a/src/UnitTest/SqlScriptPublish/SqlScriptOptionsTests.cs b/src/UnitTest/SqlScriptPublish/SqlScriptOptionsTests.cs index 72f7b5b2..c66cf8df 100644 --- a/src/UnitTest/SqlScriptPublish/SqlScriptOptionsTests.cs +++ b/src/UnitTest/SqlScriptPublish/SqlScriptOptionsTests.cs @@ -39,7 +39,8 @@ public void ConfigureVisibleEnumFields_hides_inappropriate_values() ScriptDatabaseEngineEdition.SqlServerExpressEdition, ScriptDatabaseEngineEdition.SqlServerStretchEdition, ScriptDatabaseEngineEdition.SqlServerManagedInstanceEdition, - ScriptDatabaseEngineEdition.SqlDatabaseEdgeEdition + ScriptDatabaseEngineEdition.SqlDatabaseEdgeEdition, + ScriptDatabaseEngineEdition.SqlAzureArcManagedInstanceEdition }), "Allowed edition values for SingleInstance type"); scriptOptions.TargetDatabaseEngineType = SqlScriptOptions.ScriptDatabaseEngineType.SqlAzure; editionAllowedValues = engineEditionProperty.Converter.GetStandardValues(context)