Skip to content

Commit 9168b0f

Browse files
blinkovblinkov
authored
Merge branch 'main' into hc-api-doc
2 parents 64f66af + e2fecfd commit 9168b0f

File tree

18,592 files changed

+1168774
-648600
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

18,592 files changed

+1168774
-648600
lines changed

‎.github/CODEOWNERS

+19
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,30 @@
11
/*.md @ydb-platform/docs
22

3+
/build @ydb-platform/ci
4+
/certs @ydb-platform/ci
5+
/contrib @ydb-platform/ci
6+
/devtools @ydb-platform/ci
7+
/library @ydb-platform/ci
8+
/scripts @ydb-platform/ci
9+
/tools @ydb-platform/ci
10+
/util @ydb-platform/ci
11+
/vendor @ydb-platform/ci
12+
/yt @ydb-platform/ci
13+
314
/ydb/core/fq/ @ydb-platform/fq
415
/ydb/core/public_http/ @ydb-platform/fq
516

617
/ydb/docs/ @ydb-platform/docs
718

819
/ydb/library/yql/ @ydb-platform/yql
920
/ydb/library/yql/dq @ydb-platform/yql @ydb-platform/qp
21+
/ydb/library/yql/providers/common/http_gateway @ydb-platform/fq
22+
/ydb/library/yql/providers/common/db_id_async_resolver @ydb-platform/fq
1023
/ydb/library/yql/providers/generic @ydb-platform/fq
1124
/ydb/library/yql/providers/pq @ydb-platform/fq
1225
/ydb/library/yql/providers/s3 @ydb-platform/fq
1326
/ydb/library/yql/providers/solomon @ydb-platform/fq
27+
/ydb/library/yql/udfs/common/clickhouse/client @ydb-platform/fq
1428

1529
/ydb/library/yql/yt @Krock21 @Krisha11 @zlobober @gritukan
1630

@@ -21,3 +35,8 @@
2135
/ydb/services/datastreams @ydb-platform/Topics
2236
/ydb/services/deprecated/persqueue_v0 @ydb-platform/Topics
2337
/ydb/services/persqueue_v1 @ydb-platform/Topics
38+
39+
/ydb/core/config/ut @ydb-platform/core
40+
41+
/ydb/core/viewer @ydb-platform/ui-backend
42+
/ydb/core/protos/node_whiteboard.proto @ydb-platform/ui-backend

‎.github/TESTOWNERS

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#Docs @blinkov TEAM:@ydb-platform/docs
2+
/*.md @ydb-platform/docs
3+
/ydb/docs/ @ydb-platform/docs
4+
5+
#YQL @vitstn @ydb-platform/yql
6+
/ydb/library/yql/ @ydb-platform/yql
7+
/ydb/library/yql/dq @ydb-platform/yql @ydb-platform/qp
8+
/ydb/library/yql/yt @Krock21 @Krisha11 @zlobober @gritukan
9+
10+
#Row Tables Development Team @azevaykin TEAM:@ydb-platform/datashard
11+
/ydb/core/scheme @ydb-platform/datashard
12+
/ydb/core/tablet_flat @ydb-platform/datashard
13+
/ydb/core/tx/datashard @ydb-platform/datashard
14+
/ydb/core/mon_alloc @ydb-platform/datashard
15+
/ydb/core/tx/coordinator @ydb-platform/datashard
16+
/ydb/core/statistics @ydb-platform/datashard
17+
18+
#Column Tables Development Team @zverevgeny TEAM:@ydb-platform/cs
19+
/ydb/core/tx/columnshard @ydb-platform/cs
20+
21+
22+
#Distributed System Infrastructure Team @ijon TEAM:@ydb-platform/system-infra
23+
/ydb/core/mind/hive @ydb-platform/system-infra
24+
/ydb/core/tx/schemeshard @ydb-platform/system-infra
25+
/ydb/core/tx/columnshard @ivanmorozov333 @ydb-platform/system-infra
26+
/ydb/core/util @ydb-platform/system-infra
27+
/ydb/core/client @ydb-platform/system-infra
28+
/ydb/core/engine @ydb-platform/system-infra
29+
/ydb/tests/functional/hive @ydb-platform/system-infra
30+
/ydb/core/tx/scheme_board @ydb-platform/system-infra
31+
/ydb/core/tx/scheme_cache @ydb-platform/system-infra
32+
/ydb/core/audit @ydb-platform/system-infra
33+
/ydb/core/security @ydb-platform/system-infra
34+
/ydb/core/health_check @ydb-platform/system-infra
35+
/ydb/library/aclib @ydb-platform/system-infra
36+
/ydb/mvp/oidc_proxy @ydb-platform/system-infra
37+
/ydb/tests/functional/audit @ydb-platform/system-infra
38+
/ydb/tests/functional/limits @ydb-platform/system-infra
39+
/ydb/tests/functional/scheme_shard @ydb-platform/system-infra
40+
/ydb/tests/functional/tenants @ydb-platform/system-infra
41+
/ydb/services/ydb/table_split_ut @ydb-platform/system-infra
42+
43+
#YDB Query Execution Team @gridnevvvit TEAM:@ydb-platform/qp
44+
/ydb/core/kqp @ydb-platform/qp
45+
#YDB Query Optimizer Team @pavelvelikhov TEAM:@ydb-platform/qp
46+
/ydb/tests/functional/canonical @ydb-platform/qp
47+
/ydb/tests/functional/suite_tests @ydb-platform/qp
48+
/ydb/tests/functional/clickbench @ydb-platform/qp
49+
/ydb/tests/functional/postgresql @ydb-platform/qp
50+
51+
#YDB Application Team @asmyasnikov TEAM:@ydb-platform/appteam
52+
/ydb/public/sdk @ydb-platform/appteam
53+
/ydb/tests/functional/ydb_cli @ydb-platform/appteam
54+
/ydb/public/lib/ydb_cli @ydb-platform/appteam
55+
/ydb/apps/ydb @ydb-platform/appteam
56+
57+
#YDB Engineering Team @maximyurchuk
58+
# - tbd
59+
60+
#Orphan components @spuchin @maximyurchuk @kirrysin @ydb-platform/duty
61+
/ydb/services/ydb @ydb-platform/duty
62+
/ydb/core/util @ydb-platform/duty
63+
/ydb/core/client @ydb-platform/duty
64+
/ydb/core/engine @ydb-platform/duty
65+
66+
#Distributed Storage Team @the-ancient-1 TEAM:@ydb-platform/storage
67+
/ydb/core/blobstorage @ydb-platform/storage
68+
/ydb/core/mind/bscontroller @ydb-platform/storage
69+
/ydb/library/actors @ydb-platform/storage
70+
/ydb/core/actorlib_impl @ydb-platform/storage
71+
/ydb/core/keyvalue @ydb-platform/storage
72+
/ydb/core/test_tablet @ydb-platform/storage
73+
/ydb/core/blob_depot @ydb-platform/storage
74+
/ydb/core/erasure @ydb-platform/storage
75+
76+
#Core Functionality Team @CyberROFL TEAM:@ydb-platform/core
77+
/ydb/core/config/ut @ydb-platform/core
78+
/ydb/core/cms @ydb-platform/core
79+
/ydb/tests/functional/cms @ydb-platform/core
80+
/ydb/core/quoter @ydb-platform/core
81+
82+
#Data Delivery Systems Team / LogBroker @alexnick88 TEAM:@ydb-platform/topics
83+
/ydb/tests/functional/sqs @ydb-platform/Topics
84+
/ydb/core/kafka_proxy @ydb-platform/Topics
85+
/ydb/core/persqueue @ydb-platform/Topics
86+
/ydb/services/datastreams @ydb-platform/Topics
87+
/ydb/services/deprecated/persqueue_v0 @ydb-platform/Topics
88+
/ydb/services/persqueue_v1 @ydb-platform/Topics
89+
/ydb/core/ymq @ydb-platform/Topics
90+
/ydb/core/http_proxy @ydb-platform/Topics
91+
92+
#YDB Analytics TEAM:???
93+
94+
#Federative query @uzhastik TEAM:@ydb-platform/fq
95+
/ydb/tests/fq @ydb-platform/fq
96+
/ydb/core/fq/ @ydb-platform/fq
97+
/ydb/services/fq/ @ydb-platform/fq
98+
/ydb/library/yql/providers/common/http_gateway @ydb-platform/fq
99+
/ydb/library/yql/providers/common/db_id_async_resolver @ydb-platform/fq
100+
/ydb/library/yql/providers/generic @ydb-platform/fq
101+
/ydb/library/yql/providers/pq @ydb-platform/fq
102+
/ydb/library/yql/providers/s3 @ydb-platform/fq
103+
/ydb/library/yql/providers/solomon @ydb-platform/fq
104+
/ydb/core/public_http/ @ydb-platform/fq
105+
/ydb/core/kqp/ut/federated_query @ydb-platform/fq
106+
107+
#@ydb-platform/ui-backend
108+
/ydb/core/viewer @ydb-platform/ui-backend
109+
/ydb/core/protos/node_whiteboard.proto @ydb-platform/ui-backend

‎.github/actions/build_analytics/action.yml

+14-2
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ runs:
3535
export TARGET_DIR=${{ inputs.build_target }}
3636
export build_preset="${{ inputs.build_preset }}"
3737
export build_target="${{ inputs.build_target }}"
38-
./ya tool bloat --linker-map $TARGET_DIR/$TARGET_NAME.map.lld --input $TARGET_DIR/$TARGET_NAME --save-html ya_bloat_html
38+
./ya tool bloat --linker-map $TARGET_DIR/$TARGET_NAME.map.lld --input $TARGET_DIR/$TARGET_NAME --save-html ya_bloat_html --save-json bloat.json
39+
./ydb/ci/build_bloat/template_bloat.py -j bloat.json -o template_bloat -t template_bloat_html
3940
./ydb/ci/build_bloat/main.py --build-dir . --html-dir-cpp html_cpp_impact --html-dir-headers html_headers_impact
4041
./ydb/ci/build_bloat/ydb_upload.py --html-dir-cpp html_cpp_impact --html-dir-headers html_headers_impact
4142
@@ -45,7 +46,18 @@ runs:
4546
set -ex
4647
s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "ya_bloat_html/" "$S3_BUCKET_PATH/ya_bloat_html/"
4748
echo "[ya bloat tool]($S3_URL_PREFIX/ya_bloat_html/index.html) (better use Safari, because it is too large for Chrome)" >> $GITHUB_STEP_SUMMARY
48-
49+
50+
s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "template_bloat.by_size.txt" "$S3_BUCKET_PATH/template_bloat.by_size.txt"
51+
TEMPLATE_BLOAT_BY_SIZE_URL=$S3_URL_PREFIX/template_bloat.by_size.txt
52+
53+
s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "template_bloat.by_count.txt" "$S3_BUCKET_PATH/template_bloat.by_count.txt"
54+
TEMPLATE_BLOAT_BY_COUNT_URL=$S3_URL_PREFIX/template_bloat.by_count.txt
55+
56+
s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "template_bloat_html/" "$S3_BUCKET_PATH/template_bloat_html/"
57+
TEMPLATE_BLOAT_TREEMAP_URL=$S3_URL_PREFIX/template_bloat_html/index.html
58+
59+
echo "[template bloat]($TEMPLATE_BLOAT_TREEMAP_URL) ([sorted by size]($TEMPLATE_BLOAT_BY_SIZE_URL), [sorted by count]($TEMPLATE_BLOAT_BY_COUNT_URL))" >> $GITHUB_STEP_SUMMARY
60+
4961
s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_cpp_impact/" "$S3_BUCKET_PATH/html_cpp_impact/"
5062
echo "[cpp compilation time]($S3_URL_PREFIX/html_cpp_impact/index.html)" >> $GITHUB_STEP_SUMMARY
5163

‎.github/actions/build_and_test_ya/action.yml

+57-36
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,13 @@ inputs:
1414
type: boolean
1515
default: true
1616
description: "run tests"
17-
run_tests_if_build_fails:
18-
default: "true"
19-
description: "run tests if build fails"
2017
test_threads:
2118
type: string
2219
default: 28
2320
description: "Test threads count"
2421
link_threads:
2522
type: string
26-
default: 8
23+
default: 12
2724
description: "link threads count"
2825
test_size:
2926
type: string
@@ -45,6 +42,9 @@ inputs:
4542
additional_ya_make_args:
4643
type: string
4744
default: ""
45+
test_retry_count:
46+
default: ""
47+
description: "how many times to retry failed tests"
4848
secs:
4949
type: string
5050
default: ""
@@ -68,7 +68,8 @@ runs:
6868
# tricky: we are searching job with name that contains build_preset
6969
check_url=$(curl -s $jobs_url | jq --arg n "$BUILD_PRESET" -r '.jobs[] | select(.name | contains($n)) | .html_url')
7070
71-
echo "Pre-commit [check]($check_url) for $(git rev-parse HEAD) has started." | .github/scripts/tests/comment-pr.py --rewrite
71+
platform_name="$(echo "$(uname -s)-$(uname -p)" | tr '[:upper:]' '[:lower:]')-$BUILD_PRESET"
72+
echo "Pre-commit [check]($check_url) **$platform_name** for $(git rev-parse HEAD) has started." | .github/scripts/tests/comment-pr.py --rewrite
7273
7374
curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
7475
https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
@@ -80,6 +81,10 @@ runs:
8081
-d '{"state":"pending","description":"The check has been started","context":"test_${{inputs.build_preset}}"}'
8182
fi
8283
84+
- name: Clean ya cache
85+
shell: bash
86+
run: rm -rf ~/.ya
87+
8388
- name: Prepare s3cmd
8489
uses: ./.github/actions/s3cmd
8590
with:
@@ -94,8 +99,8 @@ runs:
9499
if: inputs.increment == 'true'
95100
uses: ./.github/actions/graph_compare
96101

97-
- name: Change target in case of incremental (tests)
98-
id: test_run_choice
102+
- name: Change target in case of incremental
103+
id: target_choice
99104
shell: bash
100105
run: |
101106
if [ "${{ inputs.increment }}" == "true" ]; then
@@ -104,46 +109,26 @@ runs:
104109
echo "target=${{ inputs.build_target }}" >> $GITHUB_OUTPUT
105110
fi
106111
107-
- name: Build
108-
uses: ./.github/actions/build_ya
112+
- name: Run build and tests
109113
id: build
110-
if: ${{ inputs.run_build == 'true' }}
111-
with:
112-
build_target: ${{ inputs.build_target }}
113-
build_preset: ${{ inputs.build_preset }}
114-
bazel_remote_uri: ${{ fromJSON( inputs.vars ).REMOTE_CACHE_URL || '' }}
115-
bazel_remote_username: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
116-
bazel_remote_password: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
117-
put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
118-
link_threads: ${{ inputs.link_threads }}
119-
additional_ya_make_args: ${{ inputs.additional_ya_make_args }}
120-
121-
- name: Run tests
122114
uses: ./.github/actions/test_ya
123-
if: ${{ inputs.run_tests == 'true' && (steps.build.outputs.success == 'true' || inputs.run_tests_if_build_fails == 'true') }}
124115
with:
125-
build_target: ${{ steps.test_run_choice.outputs.target }}
116+
build_target: ${{ steps.target_choice.outputs.target }}
126117
build_preset: ${{ inputs.build_preset }}
127118
test_size: ${{ inputs.test_size }}
128119
test_type: ${{ inputs.test_type }}
129-
testman_token: ${{ fromJSON( inputs.secs ).TESTMO_TOKEN }}
120+
run_tests: ${{ inputs.run_tests }}
121+
testman_token: ${{ fromJSON( inputs.secs ).TESTMO_TOKEN2 }}
130122
testman_url: ${{ fromJSON( inputs.vars ).TESTMO_URL }}
131123
testman_project_id: ${{ fromJSON( inputs.vars ).TESTMO_PROJECT_ID }}
132124
link_threads: ${{ inputs.link_threads }}
133125
additional_ya_make_args: ${{ inputs.additional_ya_make_args }}
134126
test_threads: ${{ inputs.test_threads }}
135-
136-
- name: Notify about failed build
137-
if: ${{ steps.build.outputs.success != 'true' && inputs.run_tests == 'true' && inputs.run_tests_if_build_fails == 'false' }}
138-
shell: bash
139-
run: |
140-
echo 'Build failed. See the [build log](${{ steps.build.outputs.log_url }}).' >> $GITHUB_STEP_SUMMARY
141-
142-
if [[ "$GITHUB_EVENT_NAME" =~ ^pull_request ]]; then
143-
echo "Tests run skipped." | .github/scripts/tests/comment-pr.py --fail
144-
fi
145-
146-
exit 1
127+
bazel_remote_uri: ${{ fromJSON( inputs.vars ).REMOTE_CACHE_URL || '' }}
128+
bazel_remote_username: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_USERNAME || '' }}
129+
bazel_remote_password: ${{ fromJSON( inputs.secs ).REMOTE_CACHE_PASSWORD || '' }}
130+
put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
131+
test_retry_count: ${{ inputs.test_retry_count }}
147132

148133
- name: build_stats
149134
shell: bash
@@ -152,9 +137,45 @@ runs:
152137
run: |
153138
set -x
154139
export build_preset="${{ inputs.build_preset }}"
140+
export commit_git_sha="$(git rev-parse HEAD)"
141+
155142
python3 -m pip install ydb ydb[yc]
156143
python3 .github/scripts/send_build_stats.py
157144
145+
- name: show_build_size_diff
146+
shell: bash
147+
continue-on-error: true
148+
if: always()
149+
env:
150+
GITHUB_TOKEN: ${{ github.token }}
151+
run: |
152+
set -x
153+
export build_preset="${{ inputs.build_preset }}"
154+
export branch_to_compare="$GITHUB_REF_NAME"
155+
export yellow_treshold=102400
156+
export red_treshold=2097152
157+
export commit_git_sha="$(git rev-parse HEAD)"
158+
159+
python3 -m pip install ydb ydb[yc] humanize
160+
161+
get_sizes_comment_script=.github/scripts/get_build_diff.py
162+
comment_raw=`$get_sizes_comment_script`
163+
164+
IFS=';;;'
165+
read -ra comment_arr <<< "$comment_raw"
166+
167+
printf "$comment"
168+
if [[ ${comment_raw} != "Error"* ]];then
169+
color=${comment_arr[0]}
170+
replace=$color";;;"
171+
comment=${comment_raw/$replace/""}
172+
173+
printf "$comment" | .github/scripts/tests/comment-pr.py --color $color
174+
175+
else
176+
echo "Skipped build size difference, comment_raw = ${comment_raw}"
177+
fi
178+
158179
- name: comment-if-cancel
159180
shell: bash
160181
if: cancelled() && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target')

0 commit comments

Comments
 (0)