Merge branch 'main' into KIKIMR-19522-btree-precharge-groups

Author: kunga

.github/config/muted_functest.txt

@@ -4,5 +4,4 @@ ydb/tests/functional/tenants/test_storage_config.py::TestStorageConfig::*
 ydb/tests/functional/tenants/test_tenants.py::*
 ydb/tests/functional/audit/*
 ydb/tests/functional/clickbench::test.py.test_plans*
-ydb/tests/fq/s3*
 ydb/tests/fq/yds/test_metrics_cleanup.py::TestCleanup*test_cleanup[v1]

.github/config/muted_test.txt

@@ -5,7 +5,6 @@ ydb-tests-functional-kqp-kqp_query_session/KqpQuerySession::NoLocalAttach
 ydb-core-blobstorage-ut_blobstorage/VDiskAssimilation::Test
 ydb-core-tx-columnshard-ut_schema/TColumnShardTestSchema::ForgetAfterFail
 ydb-core-tx-columnshard-ut_schema/TColumnShardTestSchema::RebootForgetAfterFail
-ydb-core-kqp-ut-federated_query-generic/*
 ydb-library-yql-sql-pg-ut/PgSqlParsingAutoparam::AutoParamValues_DifferentTypes
 ydb-core-blobstorage-ut_blobstorage/[6/10]*
 ydb/core/blobstorage/ut_blobstorage/Defragmentation::DoesItWork

.github/config/muted_ya.txt

@@ -14,17 +14,21 @@ ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
 ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
 ydb/core/kqp/ut/federated_query/generic *
 ydb/core/kqp/ut/olap *
+ydb/core/kqp/ut/pg KqpPg.CreateIndex
 ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
 ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
 ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
 ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.ForgetAfterFail
 ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.RebootForgetAfterFail
 ydb/core/tx/columnshard/engines/ut *
 ydb/core/tx/coordinator/ut Coordinator.RestoreTenantConfiguration
+ydb/core/tx/schemeshard/ut_restore TImportTests.ShouldSucceedOnManyTables
 ydb/core/tx/schemeshard/ut_split_merge TSchemeShardSplitBySizeTest.Merge1KShards
 ydb/library/actors/http/ut HttpProxy.TooLongHeader
 ydb/library/actors/http/ut sole*
+ydb/library/yql/providers/generic/connector/tests sole*
 ydb/library/yql/sql/pg/ut PgSqlParsingAutoparam.AutoParamValues_DifferentTypes
+ydb/library/yql/tests/sql/dq_file/part6 test.py.test[bigdate-input_date32-default.txt-Results]
 ydb/library/yql/tests/sql/dq_file/part16 *
 ydb/library/yql/tests/sql/dq_file/part18 test.py.test[expr-cast_type_bind-default.txt-Analyze]
 ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.WriteRead
@@ -37,18 +41,15 @@ ydb/tests/fq/generic *
 ydb/tests/fq/restarts test_insert_restarts.py.TestS3.test_atomic_upload_commit*
 ydb/tests/fq/s3 *
 ydb/tests/fq/yds test_metrics_cleanup.py.TestCleanup.test_cleanup[v1]
+ydb/tests/fq/yds test_read_rules_deletion.py.TestReadRulesDeletion.test_delete_read_rules[v1-with_recovery]
+ydb/tests/fq/yds test_select_1.py.TestSelect1.test_unwrap_null[v1]
 ydb/tests/functional/audit *
-ydb/tests/functional/blobstorage test_replication.py.TestReplicationAfterNodesRestart.test_replication[mirror-3-dc]
+ydb/tests/functional/blobstorage test_replication.py.TestReplicationAfterNodesRestart.test_replication*
 ydb/tests/functional/clickbench test.py.test_plans*
 ydb/tests/functional/clickbench test.py.test_run_*column]
 ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach
 ydb/tests/functional/postgresql test_postgres.py.TestPostgresSuite.test_postgres_suite*
 ydb/tests/functional/restarts test_restarts.py.*
-ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud*
-ydb/tests/functional/sqs/cloud test_yandex_cloud_queue_counters.py.TestYmqQueueCounters.test_purge_queue_counters
-ydb/tests/functional/sqs/common test_queue_counters.py.TestSqsGettingCounters.test_purge_queue_counters
-ydb/tests/functional/sqs/large test_leader_start_inflight.py.TestSqsMultinodeCluster.test_limit_leader_start_inflight[tables_format*]
-ydb/tests/functional/sqs/large [*/10]*
 ydb/tests/functional/sqs/merge_split_common_table/fifo *
 ydb/tests/functional/sqs/merge_split_common_table/std *
 ydb/tests/functional/tenants test_dynamic_tenants.py.*

.github/workflows/build_and_test_ya.yml

@@ -49,6 +49,9 @@ on:
       put_build_results_to_cache:
         type: boolean
         default: true
+      commit_sha:
+        type: string
+        default: ""
 defaults:
   run:
     shell: bash
@@ -57,15 +60,11 @@ jobs:
     name: Build and test ${{ inputs.build_preset }}
     runs-on: [ self-hosted, "${{ inputs.runner_label }}", "${{ inputs.runner_additional_label || inputs.runner_label }}"]
     steps:
-    - name: Checkout PR
-      uses: actions/checkout@v3
-      if: github.event.pull_request.head.sha != ''
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
     - name: Checkout
       uses: actions/checkout@v3
-      if: github.event.pull_request.head.sha == ''
-    
+      with:
+        ref: ${{ inputs.commit_sha }}
+
     - name: comment-build-start
       if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
       shell: bash
@@ -77,7 +76,7 @@ jobs:
         # tricky: we are searching job with name that contains build_preset
         check_url=$(curl -s $jobs_url | jq --arg n "$BUILD_PRESET" -r '.jobs[] | select(.name | contains($n)) | .html_url')
         
-        echo "Pre-commit [check]($check_url) for ${{ github.event.pull_request.head.sha }} has started." | .github/scripts/tests/comment-pr.py --rewrite
+        echo "Pre-commit [check]($check_url) for ${{ inputs.commit_sha }} has started." | .github/scripts/tests/comment-pr.py --rewrite
     
     - name: Prepare s3cmd
       uses: ./.github/actions/s3cmd

.github/workflows/build_and_test_ya_provisioned.yml

@@ -90,6 +90,9 @@ on:
       put_build_results_to_cache:
         type: boolean
         default: true
+      commit_sha:
+        type: string
+        default: ""
 jobs:
   main:
     uses: ./.github/workflows/build_and_test_ya.yml
@@ -107,4 +110,5 @@ jobs:
       link_threads: ${{ inputs.link_threads }}
       test_threads: ${{ inputs.test_threads }}
       put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
+      commit_sha: ${{ inputs.commit_sha }}
     secrets: inherit

.github/workflows/pr_check.yml

@@ -1,5 +1,5 @@
 name: PR-check
-on: 
+on:
   pull_request_target:
     branches:
       - 'main'
@@ -20,7 +20,8 @@ jobs:
     if: ${{vars.CHECKS_SWITCH != '' && fromJSON(vars.CHECKS_SWITCH).pr_check == true}}
     runs-on: ubuntu-latest
     outputs:
-      result: ${{ steps.check-ownership-membership.outputs.result }}
+      result: ${{ steps.check-ownership-membership.outputs.result == 'true' && steps.check-is-mergeable.outputs.result == 'true' }}
+      commit_sha: ${{ steps.check-is-mergeable.outputs.commit_sha }}
     steps:
       - name: Check if running tests is allowed
         id: check-ownership-membership
@@ -32,17 +33,17 @@ jobs:
             const okToTestLabel = labels.find(
               label => label.name == 'ok-to-test'
             );
-
+            
             console.log("okToTestLabel=%o", okToTestLabel !== undefined);
-
+            
             if (okToTestLabel !== undefined) {
               return true;
             }
-
+            
             // This is used primarily in forks. Repository owner
             // should be allowed to run anything.
             const userLogin = context.payload.pull_request.user.login;
-
+            
             // How to interpret membership status code:
             // https://docs.github.com/rest/collaborators/collaborators#check-if-a-user-is-a-repository-collaborator
             const isRepoCollaborator = async function () {
@@ -60,12 +61,12 @@ jobs:
                 throw error;
               }
             }
-
+            
             if (context.payload.repository.owner.login == userLogin) {
               console.log("You are the repository owner!");
               return true;
             }
-
+            
             if (await isRepoCollaborator()) {
               console.log("You are a collaborator!");
               return true;
@@ -88,26 +89,100 @@ jobs:
         uses: actions/github-script@v6
         with:
           script: |
-            const { owner, repo } = context.repo;
+            let labelsToRemove = ['ok-to-test', 'rebase-and-check'];
             const prNumber = context.payload.pull_request.number;
-            const labelToRemove = 'ok-to-test';
-            try {
-              const result = await github.rest.issues.removeLabel({
-                owner,
-                repo,
-                issue_number: prNumber,
-                name: labelToRemove
-              });
-            } catch(e) {
-              // ignore the 404 error that arises
-              // when the label did not exist for the
-              // organization member
-              console.log(e);
+            const prLabels = new Set(context.payload.pull_request.labels.map(l => l.name));
+            for await (const label of labelsToRemove.filter(l => prLabels.has(l))) {
+              core.info(`remove label=${label} for pr=${prNumber}`);
+              try {
+                const result = await github.rest.issues.removeLabel({
+                  ...context.repo,
+                  issue_number: prNumber,
+                  name: label
+                });
+              } catch(error) {
+                // ignore the 404 error that arises
+                // when the label did not exist for the
+                // organization member
+                if (error.status && error.status != 404) {
+                  throw error;
+                }
+              }
+            }
+      - name: check is mergeable
+        id: check-is-mergeable
+        if: steps.check-ownership-membership.outputs.result == 'true'
+        uses: actions/github-script@v6
+        with:
+          result-encoding: string
+          script: |
+            let pr = context.payload.pull_request;
+            const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+            const header = `<!-- merge pr=${pr.number} -->\n`;
+            
+            const fail_msg = header + ':red_circle: Unable to merge your PR into the `main` branch. '
+                + 'Please rebase or merge it with the `main` branch.'
+            
+            let i = 0;
+            
+            while (pr.mergeable == null || i >= 60) {
+                console.log("get pull-request status");
+    
+                let result = await github.rest.pulls.get({
+                    ...context.repo,
+                    pull_number: pr.number
+                })
+            
+                pr = result.data;
+              
+                if (pr.mergeable == null) {
+                  await delay(5000);
+                }
+            
+                i += 1;
+            }
+            
+            console.log("pr.mergeable=%o", pr.mergeable);
+            
+            const { data: comments } = await github.rest.issues.listComments({
+                issue_number: context.issue.number,
+                owner: context.repo.owner,
+                repo: context.repo.repo
+            });
+            
+            const commentToUpdate = comments.find(comment => comment.body.startsWith(header));
+            
+            if (pr.mergeable === false) {
+                let commentParams = {
+                    ...context.repo,
+                    issue_number: context.issue.number,
+                    body: fail_msg
+                };
+            
+                if (commentToUpdate) {
+                    await github.rest.issues.updateComment({
+                        ...commentParams,
+                        comment_id: commentToUpdate.id,
+                    });
+                } else {
+                    await github.rest.issues.createComment({...commentParams});
+                }
+                core.setFailed("Merge conflict detected");
+                return false;
+            } else if (commentToUpdate) {
+                await github.rest.issues.deleteComment({
+                    ...context.repo,
+                    issue_number: context.issue.number,
+                    comment_id: commentToUpdate.id,
+                });
             }
+            core.info(`commit_sha=${pr.commit_sha}`);
+            core.setOutput('commit_sha', pr.merge_commit_sha);
+            return true;
   build_and_test:
     needs:
       - check-running-allowed
-    if: needs.check-running-allowed.outputs.result == 'true'
+    if: needs.check-running-allowed.outputs.result == 'true' && needs.check-running-allowed.outputs.commit_sha != ''
     strategy:
       fail-fast: false
       matrix:
@@ -122,4 +197,5 @@ jobs:
       test_threads: 52
       runner_label: auto-provisioned
       put_build_results_to_cache: true
+      commit_sha: ${{ needs.check-running-allowed.outputs.commit_sha }}
     secrets: inherit

ydb/apps/dstool/lib/dstool_cmd_cluster_set.py

@@ -24,6 +24,9 @@ def add_options(p):
     g.add_argument('--pdisk-space-color-border', choices=choices, help='PDisk space color border')
     choices = kikimr_bsconfig.TSerialManagementStage.E.keys()
     g.add_argument('--disk-management-mode', type=str, choices=choices, help='Disk management mode')
+    g.add_argument('--enable-self-heal-local-policy', action='store_const', const=True, dest='self_heal_local_policy', help='Enable SelfHeal local policy for cluster')
+    g.add_argument('--disable-self-heal-local-policy', action='store_const', const=False, dest='self_heal_local_policy', help='Disable SelfHeal local policy for cluster')
+    
     common.add_basic_format_options(p)
 
 
@@ -63,6 +66,8 @@ def create_request(args):
         cmd.MaxScrubbedDisksAtOnce.append(args.max_scrubbed_disks_at_once)
     if args.pdisk_space_color_border is not None:
         cmd.PDiskSpaceColorBorder.append(disk_color.TPDiskSpaceColor.E.Value(args.pdisk_space_color_border))
+    if args.self_heal_local_policy is not None:
+        cmd.UseSelfHealLocalPolicy.append(args.self_heal_local_policy)
 
     return request
 

ydb/ci/rightlib/increment.sh

@@ -1,6 +1,6 @@
 # Usage: cherrypick.sh <ydb_repo_root>
-# Expects previous github SHA in <ydb_repo_root>/library_import.txt
-# Reads new commits from the 'rightlib' branch, cheery-picks them to a new branch, and creates PR
+# Expects previous github SHA in <ydb_repo_root>/library/rightlib_sha.txt
+# Reads new commits from the 'rightlib' branch, cheery-picks them to the current branch, updates library/rightlib_sha.txt in the working tree
 
 set -e
 set -o pipefail
@@ -36,3 +36,5 @@ rsync -r $LIB_ROOT/ya $LIB_ROOT/build $LIB_ROOT/certs $LIB_ROOT/cmake $LIB_ROOT/
   --filter '- **/a.yaml' --filter '- **/.arcignore' --filter '- **/.yandex_meta/' --filter '- contrib/ydb/' --filter '- build/internal/' --filter '- build/ext_mapping.conf.json' \
   --filter '- **/CMakeLists*.txt' --delete
 
+echo ${newsha} > ${shapath}
+

ydb/core/actorlib_impl/ut/ya.make

@@ -19,7 +19,7 @@ PEERDIR(
     library/cpp/svnversion
     ydb/core/testlib/actors
     ydb/core/testlib/basics/default
-    ydb/library/yql/minikql/comp_nodes/llvm
+    ydb/library/yql/minikql/comp_nodes/llvm14
 )
 
 SRCS(

ydb/core/base/statestorage.cpp

@@ -136,7 +136,6 @@ ui32 TStateStorageInfo::ContentHash() const {
 void TStateStorageInfo::TSelection::MergeReply(EStatus status, EStatus *owner, ui64 targetCookie, bool resetOld) {
     ui32 unknown = 0;
     ui32 ok = 0;
-    ui32 noinfo = 0;
     ui32 outdated = 0;
 
     const ui32 majority = Sz / 2 + 1;
@@ -160,7 +159,6 @@ void TStateStorageInfo::TSelection::MergeReply(EStatus status, EStatus *owner, u
             ++ok;
             break;
         case StatusNoInfo:
-            ++noinfo;
             break;
         case StatusOutdated:
             ++outdated;

ydb/core/base/ut_board_subscriber/ya.make

@@ -13,7 +13,7 @@ PEERDIR(
     ydb/core/testlib/basics
     ydb/core/base
     ydb/core/testlib/basics/default
-    ydb/library/yql/minikql/comp_nodes/llvm
+    ydb/library/yql/minikql/comp_nodes/llvm14
 )
 
 YQL_LAST_ABI_VERSION()

ydb/core/blobstorage/dsproxy/dsproxy_blackboard.h

@@ -108,9 +108,9 @@ struct TBlobState {
 };
 
 struct TDiskGetRequest {
-    const TLogoBlobID Id;
-    const ui32 Shift;
-    const ui32 Size;
+    TLogoBlobID Id;
+    ui32 Shift;
+    ui32 Size;
     ssize_t PartMapIndex = -1;
 
     TDiskGetRequest(const TLogoBlobID &id, const ui32 shift, const ui32 size)
@@ -127,7 +127,7 @@ struct TDiskPutRequest {
         ReasonInitial,
         ReasonAccelerate
     };
-    const TLogoBlobID Id;
+    TLogoBlobID Id;
     TRope Buffer;
     EPutReason Reason;
     bool IsHandoff;

ydb/core/blobstorage/dsproxy/dsproxy_discover.cpp

@@ -499,7 +499,6 @@ class TBlobStorageGroupDiscoverRequest : public TBlobStorageGroupRequestActor<TB
         // Feed more data when possible
         bool isAllRead = true;
         bool isAllDisksAbleToStep = true;
-        ui32 unableToStepDiskCount = 0;
         bool isFirst = true;
         TLogoBlobID stepToId;
 
@@ -515,7 +514,6 @@ class TBlobStorageGroupDiscoverRequest : public TBlobStorageGroupRequestActor<TB
             } else {
                 if (!curVDisk.IsError && curVDisk.IsMoreRequested) {
                     isAllDisksAbleToStep = false;
-                    ++unableToStepDiskCount;
                 }
             }
             if (!curVDisk.IsError && !curVDisk.IsAllRead) {