Skip to content

Commit 9ef4473

Browse files
serbel324serbel324
authored
Merge branch 'main' into bs/vdisk/split-synclog-processing
2 parents bd404e9 + 47a95c5 commit 9ef4473

File tree

1,643 files changed

+200836
-22406
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,643 files changed

+200836
-22406
lines changed

‎.github/actions/build_and_test_ya/action.yml

+11-1
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,17 @@ runs:
130130
fi
131131
132132
exit 1
133-
133+
134+
- name: build_stats
135+
shell: bash
136+
continue-on-error: true
137+
if: always()
138+
run: |
139+
set -x
140+
export build_preset="${{ inputs.build_preset }}"
141+
python3 -m pip install ydb ydb[yc]
142+
python3 .github/scripts/send_build_stats.py
143+
134144
- name: comment-if-cancel
135145
shell: bash
136146
if: cancelled() && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target')

‎.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
name: setup_ci_ydb_service_account_key_file_credentials
2+
description: setup environment for access to ydb instance in cloud
3+
inputs:
4+
ci_ydb_service_account_key_file_credentials:
5+
required: false
6+
description: "token for access"
7+
runs:
8+
using: "composite"
9+
steps:
10+
- name: drop file and set environment variable
11+
if: "${{ inputs.ci_ydb_service_account_key_file_credentials != '' }}"
12+
shell: bash
13+
run: |
14+
set -eu
15+
export CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json
16+
cat << EOF > $CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS
17+
${{ inputs.ci_ydb_service_account_key_file_credentials}}
18+
EOF
19+
20+
echo "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV

‎.github/config/muted_ya.txt

+2
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
2828
ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
2929
ydb/core/kqp/ut/service [38/50]*
3030
ydb/core/persqueue/ut TPQTest.*DirectRead*
31+
ydb/core/persqueue/ut TopicAutoscaling.PartitionSplit_ManySession_NewSDK
3132
ydb/core/tx/coordinator/ut Coordinator.RestoreTenantConfiguration
3233
ydb/core/tx/datashard/ut_change_exchange Cdc.InitialScanDebezium
3334
ydb/core/tx/schemeshard/ut_restore TImportTests.ShouldSucceedOnManyTables
@@ -48,6 +49,7 @@ ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.WriteRead
4849
ydb/public/sdk/cpp/client/ydb_topic/ut TSettingsValidation.TestDifferentDedupParams
4950
ydb/public/sdk/cpp/client/ydb_topic/ut [0/10]*
5051
ydb/public/sdk/cpp/client/ydb_topic/ut [6/10]*
52+
ydb/public/sdk/cpp/client/ydb_topic/ut TxUsage::WriteToTopic_Demo_*
5153
ydb/services/datastreams/ut DataStreams.TestGetRecordsStreamWithSingleShard
5254
ydb/services/datastreams/ut DataStreams.TestPutRecordsWithRead
5355
ydb/services/datastreams/ut DataStreams.TestReservedConsumersMetering

‎.github/scripts/send_build_stats.py

+151
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
#!/usr/bin/env python3
2+
3+
import datetime
4+
import os
5+
import ydb
6+
import uuid
7+
import subprocess
8+
9+
10+
YDBD_PATH = "ydb/apps/ydbd/ydbd"
11+
12+
FROM_ENV_COLUMNS = [
13+
"github_head_ref",
14+
"github_workflow",
15+
"github_workflow_ref",
16+
"github_sha",
17+
"github_repository",
18+
"github_event_name",
19+
"github_ref_type",
20+
"github_ref_name",
21+
"github_ref",
22+
]
23+
24+
STRING_COLUMNS = FROM_ENV_COLUMNS + [
25+
"id",
26+
"git_commit_message",
27+
"binary_path",
28+
"build_preset",
29+
]
30+
31+
DATETIME_COLUMNS = [
32+
"git_commit_time",
33+
]
34+
35+
UINT64_COLUMNS = [
36+
"size_bytes",
37+
"size_stripped_bytes",
38+
]
39+
40+
ALL_COLUMNS = STRING_COLUMNS + DATETIME_COLUMNS + UINT64_COLUMNS
41+
42+
43+
def sanitize_str(s):
44+
# YDB SDK expects bytes for 'String' columns
45+
if s is None:
46+
s = "N\A"
47+
return s.encode("utf-8")
48+
49+
50+
def main():
51+
if "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ:
52+
print("Env variable CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping")
53+
return 1
54+
55+
# Do not set up 'real' variable from gh workflows because it interfere with ydb tests
56+
# So, set up it locally
57+
os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = os.environ["CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"]
58+
59+
if not os.path.exists(YDBD_PATH):
60+
# can be possible due to incremental builds and ydbd itself is not affected by changes
61+
print("{} not exists, skipping".format(YDBD_PATH))
62+
return 1
63+
64+
with ydb.Driver(
65+
endpoint="grpcs://ydb.serverless.yandexcloud.net:2135",
66+
database="/ru-central1/b1ggceeul2pkher8vhb6/etn6d1qbals0c29ho4lf",
67+
credentials=ydb.credentials_from_env_variables()
68+
) as driver:
69+
driver.wait(timeout=10, fail_fast=True)
70+
session = ydb.retry_operation_sync(
71+
lambda: driver.table_client.session().create()
72+
)
73+
with session.transaction() as tx:
74+
text_query_builder = []
75+
for type_ in STRING_COLUMNS:
76+
text_query_builder.append("DECLARE ${} as String;".format(type_))
77+
for type_ in UINT64_COLUMNS:
78+
text_query_builder.append("DECLARE ${} as Uint64;".format(type_))
79+
for type_ in DATETIME_COLUMNS:
80+
text_query_builder.append("DECLARE ${} as Datetime;".format(type_))
81+
82+
text_query_builder.append(
83+
"""INSERT INTO binary_size
84+
(
85+
{}
86+
)
87+
VALUES
88+
(
89+
{}
90+
);
91+
""".format(
92+
", \n ".join(ALL_COLUMNS),
93+
", \n ".join(["$" + column for column in ALL_COLUMNS]),
94+
)
95+
)
96+
97+
text_query = "\n".join(text_query_builder)
98+
99+
prepared_query = session.prepare(text_query)
100+
101+
binary_size_bytes = subprocess.check_output(
102+
["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)]
103+
)
104+
binary_size_stripped_bytes = subprocess.check_output(
105+
["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)]
106+
)
107+
108+
build_preset = os.environ.get("build_preset", None)
109+
github_sha = os.environ.get("GITHUB_SHA", None)
110+
111+
if github_sha is not None:
112+
git_commit_time_bytes = subprocess.check_output(
113+
["git", "show", "--no-patch", "--format=%cI", github_sha]
114+
)
115+
git_commit_message_bytes = subprocess.check_output(
116+
["git", "log", "--format=%s", "-n", "1", github_sha]
117+
)
118+
git_commit_time = datetime.datetime.fromisoformat(
119+
git_commit_time_bytes.decode("utf-8").strip()
120+
)
121+
git_commit_message = git_commit_message_bytes.decode("utf-8").strip()
122+
git_commit_time_unix = int(git_commit_time.timestamp())
123+
else:
124+
git_commit_time = None
125+
git_commit_message = None
126+
git_commit_time_unix = 0
127+
128+
parameters = {
129+
"$id": sanitize_str(str(uuid.uuid4())),
130+
"$build_preset": sanitize_str(build_preset),
131+
"$binary_path": sanitize_str(YDBD_PATH),
132+
"$size_stripped_bytes": int(binary_size_stripped_bytes.decode("utf-8")),
133+
"$size_bytes": int(binary_size_bytes.decode("utf-8")),
134+
"$git_commit_time": git_commit_time_unix,
135+
"$git_commit_message": sanitize_str(git_commit_message),
136+
}
137+
138+
for column in FROM_ENV_COLUMNS:
139+
value = os.environ.get(column.upper(), None)
140+
parameters["$" + column] = sanitize_str(value)
141+
142+
print("Executing query:\n{}".format(text_query))
143+
print("With parameters:")
144+
for k, v in parameters.items():
145+
print("{}: {}".format(k, v))
146+
147+
tx.execute(prepared_query, parameters, commit_tx=True)
148+
149+
150+
if __name__ == "__main__":
151+
exit(main())

‎.github/workflows/acceptance_run.yml

+6-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ on:
2525

2626
jobs:
2727
main:
28-
name: Build and test ydb/tests/acceptance
28+
name: Build and test ydb/tests/acceptance
2929
runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ]
3030
steps:
3131
- name: Checkout
@@ -38,6 +38,11 @@ jobs:
3838
with:
3939
ssh-private-key: ${{ secrets.SLICE_QA_SSH_PRIVATE_KEY }}
4040

41+
- name: Setup ydb access
42+
uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
43+
with:
44+
ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
45+
4146
- name: Build and test
4247
uses: ./.github/actions/build_and_test_ya
4348
with:

‎.github/workflows/build_and_test_ya.yml

+5
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,11 @@ jobs:
6565
with:
6666
ref: ${{ inputs.commit_sha }}
6767

68+
- name: Setup ydb access
69+
uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
70+
with:
71+
ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
72+
6873
- name: Build and test
6974
uses: ./.github/actions/build_and_test_ya
7075
with:

‎.github/workflows/postcommit_asan.yml

+4
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ jobs:
2020
uses: actions/checkout@v3
2121
with:
2222
fetch-depth: 2
23+
- name: Setup ydb access
24+
uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
25+
with:
26+
ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
2327
- name: Build and test
2428
uses: ./.github/actions/build_and_test_ya
2529
with:

‎.github/workflows/postcommit_relwithdebinfo.yml

+4
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ jobs:
1919
uses: actions/checkout@v3
2020
with:
2121
fetch-depth: 2
22+
- name: Setup ydb access
23+
uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
24+
with:
25+
ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
2226
- name: Build and test
2327
uses: ./.github/actions/build_and_test_ya
2428
with:

‎README.md

+25-31
Original file line numberDiff line numberDiff line change
@@ -22,73 +22,67 @@ YDB is an open source Distributed SQL Database that combines high availability a
2222

2323
## Main YDB Advantages
2424

25-
YDB is designed from scratch as a response to growing demand for scalable interactive web services. Scalability, strict consistency and effective cross-row transactions were a must for such OLTP-like workload. YDB is built by people with strong background in databases and distributed systems, who had an experience of developing No-SQL database and the Map-Reduce system for one of the largest search engines in the world.
26-
We found that YDB's flexible design allows us to build more services on top of it including persistent queues and virtual block devices.
25+
YDB was designed from scratch to respond to the growing demand for scalable interactive web services. Scalability, strict consistency, and effective cross-row transactions were a must for such an OLTP-like workload. YDB was built by people with strong backgrounds in databases and distributed systems who have experience developing a NoSQL database and the MapReduce system for one of the largest search engines in the world.
2726

2827
Basic YDB features:
2928

30-
- Fault-tolerant configuration that survive disk, node, rack or even datacenter outage;
31-
- Horizontal scalability;
32-
- Automatic disaster recovery with minimum latency disruptions for applications;
33-
- SQL dialect (YQL) for data manipulation and scheme definition;
34-
- ACID transactions across multiple nodes and tables with strict consistency.
29+
- Both row-oriented and column-oriented [tables](https://ydb.tech/docs/en/concepts/datamodel/table) for transactional and analytical workloads. Also, [persistent queues (topics)](https://ydb.tech/docs/en/concepts/topic) for moving data around.
30+
- Fault-tolerant configuration that survives disk, node, rack, or even datacenter outages.
31+
- Automatic disaster recovery with minimum latency disruptions for applications.
32+
- Independent horizontal scalability of storage and compute layers.
33+
- ACID transactions across multiple nodes and tables with strict consistency.
34+
- Rich [SQL dialect (YQL)](https://ydb.tech/docs/en/yql/reference/) for data manipulation and schema definition.
35+
- [PostgreSQL-compatible mode](https://ydb.tech/docs/en/postgresql/intro) for table operations and [Kafka-compatible mode](https://ydb.tech/docs/en/reference/kafka-api/) for topics.
36+
- YDB clusters can be deployed with [Ansible](https://ydb.tech/docs/en/devops/ansible/), [Kubernetes](https://ydb.tech/docs/en/devops/kubernetes/), or [manually](https://ydb.tech/docs/en/devops/manual/).
3537

3638
### Fault-tolerant Configurations
3739

38-
YDB could be deployed in three availability zones. Cluster remains available for both reads and writes during complete outage of a single zone. Availability zones and regions are covered in more detail [in documentation](https://ydb.tech/en/docs/concepts/databases#regions-az).
40+
YDB can be deployed in three availability zones (datacenters). A cluster remains available for both reads and writes during a complete outage of a single zone. Availability zones and regions are covered in more detail [in documentation](https://ydb.tech/en/docs/concepts/databases#regions-az).
3941

4042
### Horizontal Scalability
4143

42-
Unlike traditional relational databases YDB [scales out](https://en.wikipedia.org/wiki/Scalability#Horizontal_or_scale_out) providing developers with capability to simply extend cluster with computation or storage resources to handle increasing load. YDB has desaggregated storage and compute layers which allow you to scale storage and compute resources independently.
44+
Unlike traditional relational databases, YDB [scales out](https://en.wikipedia.org/wiki/Scalability#Horizontal_or_scale_out), providing developers with the capability to simply extend clusters with computation or storage resources to handle increasing load. YDB has disaggregated storage and compute layers, which allow you to scale storage and compute resources independently.
4345

44-
Current production installations have more than 10,000 nodes, store petabytes of data and handle millions distributed transactions per second.
46+
Current production installations have over 10000 nodes, store petabytes of data, and handle millions of distributed transactions per second.
4547

4648
### Automatic Disaster Recovery
4749

48-
YDB has built-in automatic recovery support to survive a hardware failure. After unpredictable disk, node, rack or even datacenter failure YDB remains fully available for reads and writes and restores required data redundancy automatically.
50+
YDB's built-in automatic recovery support allows it to seamlessly survive hardware failures. After unpredictable disk, node, rack, or even datacenter failure, YDB remains fully available for reads and writes and automatically restores required data redundancy.
4951

5052
### Multitenant and Serverless Database
51-
YDB has support for multitenant and serverless setups. A user can run a YDB cluster and create several databases that share one pool of storage and have different compute nodes. Alternatively a user can run several serverless databases that share one pool of compute resources to utilize them effectively.
53+
54+
YDB supports multitenant and serverless setups. A user can run a YDB cluster and create several databases that share one pool of storage and have different compute nodes. Alternatively, a user can run several serverless databases that share one pool of compute resources to utilize them effectively.
5255

5356
## Supported Platforms
5457

5558
### Minimal system requirements
5659

57-
YDB runs on x86 64bit platforms with minimum 8 GB of RAM.
60+
YDB runs on x86 64-bit platforms with at least 8 GB of RAM.
5861

5962
### Operating Systems
6063

61-
We have major experience running production systems on 64-bit x86 machines working under Ubuntu Linux.
64+
In most production environments, YDB runs on 64-bit x86 machines working under Ubuntu Linux.
6265

63-
For development purposes we test that YDB could be built and run under latest versions of MacOS and Microsoft Windows on a regular basis.
66+
For development purposes, it is regularly tested that YDB can be compiled and run under the latest versions of MacOS and Microsoft Windows.
6467

6568
## Getting Started
6669

67-
1. Install YDB using [pre-built executables](https://ydb.tech/en/docs/getting_started/self_hosted/ydb_local), [build it from source](BUILD.md) or [use Docker container](https://ydb.tech/en/docs/getting_started/self_hosted/ydb_docker).
68-
1. Install [command line interface](https://ydb.tech/en/docs/getting_started/cli) tool to work with scheme and run queries.
69-
1. Start [local cluster](https://ydb.tech/en/docs/getting_started/self_hosted/ydb_local) or container and run [YQL query](https://ydb.tech/en/docs/yql/reference/) via [YDB CLI](https://ydb.tech/en/docs/getting_started/cli).
70-
1. Access [Embedded UI](https://ydb.tech/en/docs/maintenance/embedded_monitoring/) via browser for schema navigation, query execution and other database development related tasks.
71-
1. Run available [example application](https://ydb.tech/en/docs/reference/ydb-sdk/example/go/).
72-
1. Develop an application using [YDB SDK](https://ydb.tech/en/docs/reference/ydb-sdk/).
70+
If you want to experiment with YDB, start with the [Quick Start guide](https://ydb.tech/docs/en/quickstart). It will yield a single-node cluster suitable for functional testing, app development, and similar tasks.
7371

74-
## How to Build from Source Code
75-
* Build server (ydbd) and client (ydb) binaries [from source code](BUILD.md).
72+
Suppose you want to jump into more serious scenarios like testing YDB fault tolerance, running performance benchmarks, or even running production or preproduction workloads. In that case, you'll need a full-fledged multi-node YDB cluster that can be deployed with either [Ansible](https://ydb.tech/docs/en/devops/ansible/initial-deployment) for bare metal or virtual machines or [Kubernetes](https://ydb.tech/docs/en/devops/kubernetes/initial-deployment) for containers.
7673

77-
## How to Deploy
74+
## How to Build from Source Code
7875

79-
* Deploy a cluster [using Kubernetes](https://ydb.tech/en/docs/deploy/orchestrated/concepts).
80-
* Deploy a cluster using [pre-built executables](https://ydb.tech/en/docs/getting_started/self_hosted/ydb_local).
76+
Instructions on how to build YDB server (ydbd) and client (ydb) binaries are provided in [BUILD.md](BUILD.md). Also, see documentation on [Ya Make build system](https://ydb.tech/docs/en/contributor/build-ya).
8177

8278
## How to Contribute
8379

84-
We are glad to welcome new contributors!
80+
We are glad to welcome new contributors! The [contributor's guide](CONTRIBUTING.md) provides more details on how to get started as a contributor.
8581

86-
1. Please read the [contributor's guide](CONTRIBUTING.md).
87-
2. We can accept your work to YDB after you have signed contributor's license agreement (aka CLA).
88-
3. Please don't forget to add a note to your pull request, that you agree to the terms of the CLA.
82+
There's also a separate section of [YDB documentation for contributors](https://ydb.tech/docs/en/contributor/), mostly with more technical content.
8983

9084
## Success Stories
9185

92-
Take a look at YDB [web site](https://ydb.tech/) for the latest success stories and user scenarios.
86+
Visit YDB [website](https://ydb.tech/) for the latest success stories and user scenarios.
9387

9488

‎build/conf/compilers/gnu_compiler.conf

+2-4
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ when ($FORCE_NO_PIC == "yes") {
3838
}
3939
}
4040

41-
C_COMPILER=${quo:C_COMPILER_UNQUOTED}
4241
OPTIMIZE=$_OPTIMIZE_RELEASE
4342
FSTACK=-fstack-protector
4443
DUMP_DEPS=
@@ -76,7 +75,6 @@ when ($OS_EMSCRIPTEN == "yes") {
7675
CFLAGS+=$_C_FLAGS $DEBUG_INFO_FLAGS $_C_FOPTIONS $C_WARNING_OPTS $GCC_PREPROCESSOR_OPTS $USER_CFLAGS $USER_CFLAGS_GLOBAL
7776
CXXFLAGS+=$CFLAGS $_STD_CXX $CXX_WARNING_OPTS $USER_CXXFLAGS $USER_CXXFLAGS_GLOBAL
7877
CONLYFLAGS+=$USER_CONLYFLAGS $USER_CONLYFLAGS_GLOBAL
79-
CXX_COMPILER=${quo:CXX_COMPILER_UNQUOTED}
8078
NOGCCSTACKCHECK=yes
8179
SFDL_FLAG=$_SFDL_FLAGS -o $SFDL_TMP_OUT
8280
WERROR_FLAG=-Werror
@@ -194,7 +192,7 @@ _CPP_ARGS=\
194192
$CLANG_TIDY_ARGS \
195193
$YNDEXER_ARGS \
196194
$RETRY_ARGS \
197-
$CXX_COMPILER \
195+
$CXX_COMPILER_OLD \
198196
$C_FLAGS_PLATFORM \
199197
$GCC_COMPILE_FLAGS \
200198
$CXXFLAGS \
@@ -240,7 +238,7 @@ _C_ARGS=\
240238
$CLANG_TIDY_ARGS \
241239
$YNDEXER_ARGS \
242240
$RETRY_ARGS \
243-
$C_COMPILER \
241+
$C_COMPILER_OLD \
244242
$C_FLAGS_PLATFORM \
245243
$GCC_COMPILE_FLAGS \
246244
$CFLAGS \

0 commit comments

Comments
 (0)