remove unused

Author: MrLolthe1st

.github/actions/build_ya/action.yml

@@ -85,7 +85,7 @@ runs:
         echo "::debug::get version"
         ./ya --version
         
-        ./ya make -k --build "${build_type}" --force-build-depends -D'BUILD_LANGUAGES=CPP PY3 PY2 GO' -T --stat  \
+        ./ya make -k --build "${build_type}" --force-build-depends -D'BUILD_LANGUAGES=CPP PY3 PY2 GO' -T --stat -DCONSISTENT_DEBUG \
           --log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \
           --cache-size 512G --link-threads "${{ inputs.link_threads }}" \
           "${extra_params[@]}" || (

.github/actions/test_ya/action.yml

@@ -82,6 +82,74 @@ runs:
       shell: bash
       run: npm install -g @testmo/testmo-cli
 
+    - name: Upload tests result to testmo
+      id: th
+      if: inputs.testman_token
+      shell: bash
+      env:
+        PR_NUMBER: ${{ github.event.number }}
+      run: |
+        RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+        BRANCH_TAG="$GITHUB_REF_NAME"
+        ARCH="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
+        
+        BUILD_PRESET="${{ inputs.build_preset }}"
+        case "$BUILD_PRESET" in
+          relwithdebinfo)
+            TESTMO_SOURCE="ya-${ARCH}"
+            ;;
+          debug)
+            TESTMO_SOURCE="ya-${ARCH}-debug"
+            ;;
+          release-*)
+            TESTMO_SOURCE="ya-${ARCH}-${BUILD_PRESET/release-/}"
+            ;;
+          *)
+            echo "Invalid preset: ${{ inputs.build_preset }}"
+            exit 1
+            ;;
+        esac
+        
+        case $GITHUB_EVENT_NAME in
+          workflow_dispatch)
+            TESTMO_RUN_NAME="${{ github.run_id }} manual"
+            EXTRA_TAG="manual"
+            ;;
+          pull_request | pull_request_target)
+            TESTMO_RUN_NAME="${{ github.run_id }} PR #${PR_NUMBER}"
+            EXTRA_TAG="pr"
+            BRANCH_TAG=""
+            ;;
+          schedule)
+            TESTMO_RUN_NAME="${{ github.run_id }} schedule"
+            EXTRA_TAG="schedule"
+            ;;
+          push)
+            TESTMO_RUN_NAME="${{ github.run_id }} POST"
+            EXTRA_TAG="post-commit"
+            ;;
+          *)
+            TESTMO_RUN_NAME="${{ github.run_id }}"
+            EXTRA_TAG=""
+            ;;
+        esac
+        
+        testmo automation:resources:add-link --name build --url "$RUN_URL" --resources testmo.json
+        testmo automation:resources:add-field --name git-sha --type string --value "${GITHUB_SHA:0:7}" --resources testmo.json
+        RUN_ID=$(
+        testmo automation:run:create --instance "$TESTMO_URL" --project-id ${{ inputs.testman_project_id }} \
+          --name "$TESTMO_RUN_NAME" --source "$TESTMO_SOURCE" --resources testmo.json \
+          --tags "$BRANCH_TAG" --tags "$EXTRA_TAG"
+        )
+        echo "runid=${RUN_ID}" >> $GITHUB_OUTPUT
+        echo "TEST_HISTORY_URL=${TESTMO_URL}/automation/runs/view/${RUN_ID}" >> $GITHUB_ENV
+
+    - name: Print test history link
+      shell: bash
+      if: inputs.testman_token
+      run: |
+        echo "10 [Test history](${TEST_HISTORY_URL})" >> $SUMMARY_LINKS
+
     - name: set environment variables required by some tests
       shell: bash
       run: |
@@ -149,7 +217,7 @@ runs:
           echo "::debug::start tests"
 
           ./ya test "${params[@]}" \
-            --bazel-remote-put --bazel-remote-username "${{ inputs.bazel_remote_username }}" --bazel-remote-password "${{ inputs.bazel_remote_password }}" \
+            --bazel-remote-put --bazel-remote-username "${{ inputs.bazel_remote_username }}" --bazel-remote-password "${{ inputs.bazel_remote_password }}" -DCONSISTENT_DEBUG \
             --log-file "$LOG_DIR/ya_log_prewarm.txt" --evlog-file "$LOG_DIR/ya_evlog_prewarm.jsonl" \
             --dist-cache-evict-bins --cache-tests --no-dir-outputs --test-node-output-limit 100000 --drop-graph-result-before-tests || (
               RC=$?
@@ -159,7 +227,7 @@ runs:
         
         echo "::debug::save tests reports"
         ./ya test "${params[@]}" \
-          --stat --log-file "$LOG_DIR/ya_log.txt" --evlog-file "$LOG_DIR/ya_evlog.jsonl" \
+          --stat --log-file "$LOG_DIR/ya_log.txt" --evlog-file "$LOG_DIR/ya_evlog.jsonl" -DCONSISTENT_DEBUG \
           --cache-tests --dist-cache-evict-bins --no-dir-outputs --test-node-output-limit 100000 --drop-graph-result-before-tests \
           --junit "$JUNIT_REPORT_XML" --output "$OUT_DIR" || (
             RC=$?
@@ -199,68 +267,10 @@ runs:
       run: |
         tar -C $JUNIT_REPORT_PARTS/.. -czf $REPORTS_ARTIFACTS_DIR/junit_parts.xml.tar.gz $(basename $JUNIT_REPORT_PARTS)
 
-    - name: Upload tests result to testmo
-      id: th
+    - name: Unit test history upload results
       if: inputs.testman_token
       shell: bash
-      env:
-        PR_NUMBER: ${{ github.event.number }}
       run: |
-        RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
-        BRANCH_TAG="$GITHUB_REF_NAME"
-        ARCH="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
-        
-        BUILD_PRESET="${{ inputs.build_preset }}"
-        case "$BUILD_PRESET" in
-          relwithdebinfo)
-            TESTMO_SOURCE="ya-${ARCH}"
-            ;;
-          debug)
-            TESTMO_SOURCE="ya-${ARCH}-debug"
-            ;;
-          release-*)
-            TESTMO_SOURCE="ya-${ARCH}-${BUILD_PRESET/release-/}"
-            ;;
-          *)
-            echo "Invalid preset: ${{ inputs.build_preset }}"
-            exit 1
-            ;;
-        esac
-        
-        case $GITHUB_EVENT_NAME in
-        workflow_dispatch)
-         TESTMO_RUN_NAME="${{ github.run_id }} manual"
-         EXTRA_TAG="manual"
-         ;;
-        pull_request | pull_request_target)
-         TESTMO_RUN_NAME="${{ github.run_id }} PR #${PR_NUMBER}"
-         EXTRA_TAG="pr"
-         BRANCH_TAG=""
-         ;;
-        schedule)
-         TESTMO_RUN_NAME="${{ github.run_id }} schedule"
-         EXTRA_TAG="schedule"
-         ;;
-        push)
-         TESTMO_RUN_NAME="${{ github.run_id }} POST"
-         EXTRA_TAG="post-commit"
-         ;;
-        *)
-         TESTMO_RUN_NAME="${{ github.run_id }}"
-         EXTRA_TAG=""
-         ;;
-        esac
-        
-        testmo automation:resources:add-link --name build --url "$RUN_URL" --resources testmo.json
-        testmo automation:resources:add-field --name git-sha --type string --value "${GITHUB_SHA:0:7}" --resources testmo.json
-        RUN_ID=$(
-        testmo automation:run:create --instance "$TESTMO_URL" --project-id ${{ inputs.testman_project_id }} \
-          --name "$TESTMO_RUN_NAME" --source "$TESTMO_SOURCE" --resources testmo.json \
-          --tags "$BRANCH_TAG" --tags "$EXTRA_TAG"
-        )
-        echo "runid=${RUN_ID}" >> $GITHUB_OUTPUT
-        echo "TEST_HISTORY_URL=${TESTMO_URL}/automation/runs/view/${RUN_ID}" >> $GITHUB_ENV
-        
         PROXY_ADDR=127.0.0.1:8888
 
         openssl req -x509 -newkey rsa:2048 \
@@ -277,16 +287,11 @@ runs:
         proxy_pid=$!
         
         NODE_TLS_REJECT_UNAUTHORIZED=0 testmo automation:run:submit-thread \
-          --instance "https://$PROXY_ADDR" --run-id "$RUN_ID" \
+          --instance "https://$PROXY_ADDR" --run-id "${{ steps.th.outputs.runid }}" \
           --results "$JUNIT_REPORT_PARTS/*.xml"
         
         kill $proxy_pid
 
-    - name: Print test history link
-      shell: bash
-      run: |
-        echo "10 [Test history](${TEST_HISTORY_URL})" >> $SUMMARY_LINKS
-
     - name: Test history run complete
       if: always() && inputs.testman_token
       shell: bash

.github/config/muted_functest.txt

@@ -1,15 +1,8 @@
-::sqs.cloud.test_yandex_cloud_mode
-::suite_tests.test_postgres
-::suite_tests.test_sql_logic
-::suite_tests.test_stream_query
-ydb/tests/functional/sqs/common/test_queue_attributes_validation.py::TestQueueAttributesValidation::*
-ydb/tests/functional/sqs/common/test_queues_managing.py::TestQueuesManagingWithPathTestQueuesManagingWithPath::*
-ydb/tests/functional/sqs/common/test_queues_managing.py::TestQueuesManagingWithTenant::*
-ydb/tests/functional/sqs/messaging/test_fifo_messaging.py::TestSqsFifoMessagingWithTenant::*
-ydb/tests/functional/sqs/messaging/test_fifo_messaging.py::TestSqsFifoMessagingWithPath::*
-ydb/tests/functional/sqs/large/test_leader_start_inflight.py::TestSqsMultinodeCluster::*
-ydb/tests/functional/sqs/with_quotas/test_quoting.py::TestSqsQuotingWithKesus::*
-ydb/tests/functional/suite_tests/test_stream_query.py::TestStreamQuery::test_sql_suite[results-window.test]
+ydb/tests/functional/postgresql::test_postgres.py.TestPostgresSuite.test_postgres_suite*
 ydb/tests/functional/tenants/test_dynamic_tenants.py::*
 ydb/tests/functional/tenants/test_storage_config.py::TestStorageConfig::*
 ydb/tests/functional/tenants/test_tenants.py::*
+ydb/tests/functional/audit/*
+ydb/tests/functional/clickbench::test.py.test_plans*
+ydb/tests/fq/s3*
+ydb/tests/fq/yds/test_metrics_cleanup.py::TestCleanup*test_cleanup[v1]

.github/config/muted_shard.txt

@@ -1,5 +1,5 @@
 ydb-core-blobstorage-pdisk-ut_2
-ydb-core-blobstorage-ut_blobstorage_9
-ydb-core-blobstorage-ut_vdisk2_0
-ydb-core-tx-schemeshard-ut_replication_reboots_0
-ydb-core-cms-ut_sentinel
+ydb-core-blobstorage-ut_blobstorage_4
+ydb-core-blobstorage-ut_blobstorage_5
+ydb-core-blobstorage-ut_blobstorage_6
+ydb-services-persqueue_v1-ut_3

.github/config/muted_test.txt

@@ -1,34 +1,12 @@
-ydb-core-blobstorage-pdisk-ut/TPDiskTest::TestMultipleLogSpliceNonceJump
 ydb-core-blobstorage-pdisk-ut/TSectorMap::*
-ydb-core-blobstorage-ut_blobstorage/CountingEvents::Get_Mirror3of4
 ydb-core-blobstorage-ut_blobstorage/SpaceCheckForDiskReassign::*
-ydb-core-blobstorage-ut_pdiskfit-ut/TPDiskFIT::*
-ydb-core-blobstorage-ut_vdisk/TBsHuge::*
-ydb-core-blobstorage-ut_vdisk/THugeMigration::RollbackMap_HugeBlobs
-ydb-core-cms-ut/TCmsTest::TestProcessingQueue
-ydb-core-cms-ut_sentinel/TSentinelTests::*
-ydb-core-cms-ut_sentinel/TSentinelTests::PDiskErrorState
-ydb-core-cms-ut_sentinel/TSentinelTests::PDiskFaultyState
-ydb-core-kqp-runtime-ut/KqpSpillingFileTests::StartError
-ydb-core-kqp-ut-service/KqpQueryService::CancelScriptExecution
-ydb-core-tx-schemeshard-ut_external_data_source/TExternalDataSourceTest::SchemeErrors
-ydb-core-tx-schemeshard-ut_reboots/IntermediateDirsReboots::CreateKesusWithIntermediateDirs
-ydb-core-tx-schemeshard-ut_replication_reboots/TReplicationWithRebootsTests::CreateDropRecreate
-ydb-library-yql-minikql-ut/TMiniKQLAllocTest::TestDeallocated
-ydb-library-yql-utils-backtrace-ut/TEST_BACKTRACE_AND_SYMBOLIZE::TEST_NO_KIKIMR
-ydb-public-sdk-cpp-client-ydb_topic-ut/Describe::Statistics
 ydb-services-ydb-sdk_sessions_pool_ut/YdbSdkSessionsPool::StressTestSync10
-ydb-tests-functional-kqp-kqp_indexes/ConsistentIndexRead::InteractiveTx
-ydb-tests-functional-kqp-kqp_indexes/KqpExtTest::SecondaryIndexSelectUsingScripting
 ydb-tests-functional-kqp-kqp_query_session/KqpQuerySession::NoLocalAttach
-ydb-library-yql-providers-yt-codec-codegen-ut/*
-ydb-library-yql-providers-yt-comp_nodes-ut/YtWriterTests::FlattenSkiff+LLVM
-ydb-library-yql-providers-yt-comp_nodes-ut/YtWriterTests::SimpleSkiff+LLVM
 ydb-core-blobstorage-ut_blobstorage/VDiskAssimilation::Test
-ydb-services-ydb-sdk_credprovider_ut/SdkCredProvider::PingFromProviderSyncDiscovery
-ydb-library-yql-minikql-codegen-ut/*
-ydb-library-yql-minikql-computation-ut/ComputationPatternCache::Smoke
-ydb-tests-functional-kqp-kqp_indexes/ConsistentIndexRead::InteractiveTx
-ydb-core-tx-replication-ydb_proxy-ut/YdbProxyTests::ReadTopic
 ydb-core-tx-columnshard-ut_schema/TColumnShardTestSchema::ForgetAfterFail
 ydb-core-tx-columnshard-ut_schema/TColumnShardTestSchema::RebootForgetAfterFail
+ydb-core-kqp-ut-federated_query-generic/*
+ydb-core-client-ut/TClientTest::FollowerOfflineBoot
+ydb-library-yql-sql-pg-ut/PgSqlParsingAutoparam::AutoParamValues_DifferentTypes
+ydb-core-blobstorage-ut_blobstorage/[6/10]*
+ydb/core/blobstorage/ut_blobstorage/Defragmentation::DoesItWork

.github/scripts/tests/transform-ya-junit.py

@@ -22,7 +22,7 @@ def add_unittest(self, fn):
         with open(fn, "r") as fp:
             for line in fp:
                 line = line.strip()
-                path, rest = line.split("/")
+                path, rest = line.split("/", maxsplit=1)
                 path = path.replace("-", "/")
                 rest = rest.replace("::", ".")
                 self.populate(f"{path}/{rest}")
@@ -101,7 +101,7 @@ def get_logs(self, cls, name):
 def filter_empty_logs(logs):
     result = {}
     for k, v in logs.items():
-        if os.stat(v).st_size == 0:
+        if not os.path.isfile(v) or os.stat(v).st_size == 0:
             continue
         result[k] = v
     return result

.github/workflows/allowed_dirs.yml

@@ -2,13 +2,10 @@ name: CheckAllowedDirs
 
 on:
   push:
-    branches: [ main ]
   pull_request:
-    branches: [ main ]
 
 jobs:
   build:
-    if: ${{ vars.CHECKS_SWITCH != '' && fromJSON(vars.CHECKS_SWITCH).allowed_dirs == true}}
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/pr_check.yml

@@ -6,7 +6,6 @@ on:
       - 'stable-*'
     paths-ignore:
       - 'ydb/docs/**'
-      - '.github/**'
       - '*'
     types:
       - 'opened'
@@ -122,5 +121,5 @@ jobs:
       test_type: "unittest,py3test,py2test,pytest"
       test_threads: 52
       runner_label: auto-provisioned
-      put_build_results_to_cache: false
+      put_build_results_to_cache: true
     secrets: inherit

.gitignore

@@ -34,6 +34,7 @@ __pycache__/
 /build-*
 
 .idea/
+.vscode/
 
 # KDevelop IDE
 *.kdev4

README.md

@@ -7,7 +7,7 @@
 ## YDB
 
 [Website](https://ydb.tech) |
-[Documentation](https://ydb.tech/en/docs) |
+[Documentation](https://ydb.tech/docs/en/) |
 [Official Repository](https://github.com/ydb-platform/ydb) |
 [Blog](https://blog-redirect.ydb.tech) |
 [YouTube](https://www.youtube.com/c/YDBPlatform) |