Skip to content

ci: build docker local-ydb docker image using ya make #934

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 17 additions & 44 deletions .github/docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,52 +4,23 @@ FROM ubuntu:22.04 AS builder
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Etc/UTC


RUN apt-get update \
&& apt-get install -y --no-install-recommends git wget gnupg lsb-release curl xz-utils tzdata \
cmake python3-dev python3-pip ninja-build antlr3 m4 libidn11-dev libaio1 libaio-dev make \
clang-12 lld-12 llvm-12 clang-14 lld-14 llvm-14 file \
&& pip3 install conan==1.59 grpcio-tools pyinstaller==5.13.2 six pyyaml packaging PyHamcrest cryptography \
&& (V=4.8.1; curl -L https://github.com/ccache/ccache/releases/download/v${V}/ccache-${V}-linux-x86_64.tar.xz | \
tar -xJ -C /usr/local/bin/ --strip-components=1 --no-same-owner ccache-${V}-linux-x86_64/ccache)

&& apt-get install -y --no-install-recommends \
python3 curl lsb-release python3-dev libc6-dev libidn11-dev libaio1 libaio-dev libgcc-11-dev binutils tree git

COPY ydb/ /ydbwork/ydb/

WORKDIR /ydbwork/

RUN --mount=type=secret,id=ccache_remote_storage \
mkdir build && cd build \
&& export CONAN_USER_HOME=/ydbwork/build \
&& export CCACHE_BASEDIR=/ydbwork/ \
&& export CCACHE_SLOPPINESS=locale \
&& export CCACHE_REMOTE_STORAGE="$(cat /run/secrets/ccache_remote_storage)" \
&& export CC=/usr/bin/clang-14 \
&& export CC_FOR_BUILD=$CC \
&& cmake -G Ninja -DCMAKE_BUILD_TYPE=Release \
-DCCACHE_PATH=/usr/local/bin/ccache \
-DCMAKE_TOOLCHAIN_FILE=../ydb/clang.toolchain \
../ydb \
&& ninja ydb/apps/ydbd/ydbd ydb/apps/ydb/all \
&& ccache -s \
&& strip ydb/apps/ydbd/ydbd \
&& strip ydb/apps/ydb/ydb \
&& mv ./ydb/apps/ydbd/ydbd / \
&& mv ./ydb/apps/ydb/ydb / \
&& /ydbd -V \
&& /ydb version \
&& cd .. && rm -rf build # for reduce cache size

WORKDIR /ydbwork/ydb/


# always use local_ydb.spec from main revision
COPY main/ydb/public/tools/local_ydb/local_ydb.spec ydb/ydb/public/tools/local_ydb/

RUN cd ydb && \
./ydb/tests/oss/launch/compile_protos.sh . ydb library/cpp/actors && \
cd ydb/public/tools/local_ydb/ && \
pyinstaller local_ydb.spec && \
./dist/local_ydb --help
RUN ./ya make -r -T \
-C ydb/apps/ydbd/ydbd \
-C ydb/apps/ydb/ydb \
-C ydb/public/tools/local_ydb/local_ydb \
-o /ydbwork/out/ \
&& /ydbwork/out/ydb/apps/ydbd/ydbd -V \
&& tree /ydbwork/out/ \
&& strip /ydbwork/out/ydb/apps/ydbd/ydbd /ydbwork/out/ydb/apps/ydb/ydb /ydbwork/out/ydb/public/tools/local_ydb/local_ydb \
&& rm -rf ~/.ya || echo 'unable to delete ~/.ya' # for reduce cache size


FROM ubuntu:22.04
Expand All @@ -63,9 +34,11 @@ RUN mkdir -p /root/ydb/bin/ \
&& echo '{"check_version":false}' > root/ydb/bin/config.json

COPY main/.github/docker/files/ /
COPY --from=builder /ydbwork/ydb/ydb/public/tools/local_ydb/dist/local_ydb /
COPY --from=builder /ydbd /ydb /

COPY --from=builder \
/ydbwork/out/ydb/apps/ydbd/ydbd \
/ydbwork/out/ydb/apps/ydb/ydb \
/ydbwork/out/ydb/public/tools/local_ydb/local_ydb \
/

# YDB grpc
EXPOSE ${GRPC_TLS_PORT:-2135}
Expand Down
1 change: 1 addition & 0 deletions .github/docker/files/initialize_local_ydb
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/bin/bash
set -x

INIT_YDB_SCRIPT=/init_ydb

Expand Down
16 changes: 9 additions & 7 deletions .github/workflows/docker_publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ on:
required: true
default: main
description: "Git branch/tag revision to build"
dockerfile_branch:
type: string
required: true
default: main
description: "Git branch for getting Dockerfile"
image_tag:
type: string
required: true
Expand All @@ -19,20 +24,19 @@ on:
type: string
required: true
default: main
description: "Git branch/tag revision to builld local_ydb"
description: "Git branch/tag revision to build local_ydb"

jobs:
build:
runs-on: [self-hosted, auto-provisioned]
steps:
- name: Checkout .github and local_ydb
- name: Checkout .github
uses: actions/checkout@v4
with:
ref: main
ref: ${{ inputs.dockerfile_branch || 'main' }}
path: main
sparse-checkout: |
.github
ydb/public/tools/local_ydb/

- name: Checkout
uses: actions/checkout@v4
Expand All @@ -47,7 +51,7 @@ jobs:
run: |
echo "SHA=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT

- name: Set up Docker Buildx
- name: Set up Docker buildx
uses: docker/setup-buildx-action@v2

- name: Log in to the Container registry
Expand Down Expand Up @@ -82,5 +86,3 @@ jobs:
provenance: false
cache-from: type=s3,name=local_ydb,region=ru-central1,bucket=${{ vars.AWS_BUCKET }},endpoint_url=${{ vars.AWS_ENDPOINT }},access_key_id=${{ secrets.AWS_KEY_ID }},secret_access_key=${{ secrets.AWS_KEY_VALUE }}
cache-to: type=s3,name=local_ydb,region=ru-central1,bucket=${{ vars.AWS_BUCKET }},endpoint_url=${{ vars.AWS_ENDPOINT }},access_key_id=${{ secrets.AWS_KEY_ID }},secret_access_key=${{ secrets.AWS_KEY_VALUE }},mode=max
secrets: |
"ccache_remote_storage=${{ vars.REMOTE_CACHE_URL && format('http://{0}{1}', secrets.REMOTE_CACHE_AUTH, vars.REMOTE_CACHE_URL) || ''}}"