Skip to content

Commit a506ceb

Browse files
waterfoulwaterfoul
committed
Add the ability to native build cargo packages
1 parent 5e9c6a1 commit a506ceb

File tree

18 files changed

+661
-103
lines changed

18 files changed

+661
-103
lines changed

‎.automation/build.py

+67-30
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@
6767
else:
6868
VERSION_URL_SEGMENT = VERSION
6969

70-
7170
MKDOCS_URL_ROOT = ML_DOC_URL_BASE + VERSION_URL_SEGMENT
7271

7372
BRANCH = "main"
@@ -404,31 +403,6 @@ def build_dockerfile(
404403
if len(gem_packages) > 0:
405404
apk_packages += ["ruby", "ruby-dev", "ruby-bundler", "ruby-rdoc"]
406405
# Replace between tags in Dockerfile
407-
# Commands
408-
replace_in_file(
409-
dockerfile,
410-
"#FROM__START",
411-
"#FROM__END",
412-
"\n".join(list(dict.fromkeys(docker_from))),
413-
)
414-
replace_in_file(
415-
dockerfile,
416-
"#ARG__START",
417-
"#ARG__END",
418-
"\n".join(list(dict.fromkeys(docker_arg))),
419-
)
420-
replace_in_file(
421-
dockerfile,
422-
"#COPY__START",
423-
"#COPY__END",
424-
"\n".join(docker_copy),
425-
)
426-
replace_in_file(
427-
dockerfile,
428-
"#OTHER__START",
429-
"#OTHER__END",
430-
"\n".join(docker_other),
431-
)
432406
# apk packages
433407
apk_install_command = ""
434408
if len(apk_packages) > 0:
@@ -440,6 +414,44 @@ def build_dockerfile(
440414
replace_in_file(dockerfile, "#APK__START", "#APK__END", apk_install_command)
441415
# cargo packages
442416
cargo_install_command = ""
417+
# Pre-building packages
418+
prebuild_list = set(cargo_packages) & {"shellcheck-sarif", "sarif-fmt"}
419+
cargo_packages = set(cargo_packages) - prebuild_list
420+
if len(prebuild_list) > 0:
421+
docker_from += [
422+
"FROM --platform=$BUILDPLATFORM alpine:3 AS cargo-build\n"
423+
+ "WORKDIR /cargo\n"
424+
+ "ENV HOME=/cargo\n"
425+
+ "USER 0\n"
426+
+ "RUN --mount=type=cache,target=/var/cache/apk,id=apk-${BUILDARCH},sharing=locked \\\n"
427+
+ " apk add --update \\\n"
428+
+ " gcc \\\n"
429+
+ " rustup \\\n"
430+
+ " bash \\\n"
431+
+ " git \\\n"
432+
+ " musl-dev \\\n"
433+
+ " llvm \\\n"
434+
+ " clang\n"
435+
+ "RUN chown 63425:63425 /cargo\n"
436+
+ "USER 63425\n"
437+
+ "ENV CC_aarch64_unknown_linux_musl=clang \\\n"
438+
+ " AR_aarch64_unknown_linux_musl=llvm-ar \\\n"
439+
+ ' CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld" \\\n'
440+
+ " CC_x86_64_unknown_linux_musl=clang \\\n"
441+
+ " AR_x86_64_unknown_linux_musl=llvm-ar \\\n"
442+
+ ' CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld"\n'
443+
+ "ARG TARGETARCH\n"
444+
+ 'RUN rustup-init -y --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")\n'
445+
+ "\n"
446+
+ "RUN --mount=type=cache,id=cargo-${TARGETARCH},sharing=locked,target=/cargo/.cargo/registry/,uid=63425 \\\n"
447+
+ " . /cargo/.cargo/env \\\n"
448+
+ f' && cargo install {" ".join(prebuild_list)} --root /tmp --target $([[ "${{TARGETARCH}}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl") \n'
449+
+ "\n"
450+
+ "FROM scratch AS cargo\n"
451+
+ "COPY --link --from=cargo-build /tmp/bin/* /bin/\n"
452+
+ f'RUN ["/bin/' + '", "--help"]\nRUN ["/bin/'.join(prebuild_list) + '", "--help"]\n'
453+
]
454+
docker_copy += [f"COPY --from=cargo /bin/* /usr/bin/"]
443455
keep_rustup = False
444456
if len(cargo_packages) > 0:
445457
rust_commands = []
@@ -542,6 +554,31 @@ def build_dockerfile(
542554
+ " \\\n ".join(list(dict.fromkeys(gem_packages)))
543555
)
544556
replace_in_file(dockerfile, "#GEM__START", "#GEM__END", gem_install_command)
557+
# Commands
558+
replace_in_file(
559+
dockerfile,
560+
"#FROM__START",
561+
"#FROM__END",
562+
"\n".join(list(dict.fromkeys(docker_from))),
563+
)
564+
replace_in_file(
565+
dockerfile,
566+
"#ARG__START",
567+
"#ARG__END",
568+
"\n".join(list(dict.fromkeys(docker_arg))),
569+
)
570+
replace_in_file(
571+
dockerfile,
572+
"#COPY__START",
573+
"#COPY__END",
574+
"\n".join(docker_copy),
575+
)
576+
replace_in_file(
577+
dockerfile,
578+
"#OTHER__START",
579+
"#OTHER__END",
580+
"\n".join(docker_other),
581+
)
545582
flavor_env = f"ENV MEGALINTER_FLAVOR={flavor}"
546583
replace_in_file(dockerfile, "#FLAVOR__START", "#FLAVOR__END", flavor_env)
547584
replace_in_file(
@@ -1399,9 +1436,9 @@ def process_type(linters_by_type, type1, type_label, linters_tables_md):
13991436
# Pre/post commands
14001437
linter_doc_md += [
14011438
f"| {linter.name}_PRE_COMMANDS | List of bash commands to run before the linter"
1402-
f"| {dump_as_json(linter.pre_commands,'None')} |",
1439+
f"| {dump_as_json(linter.pre_commands, 'None')} |",
14031440
f"| {linter.name}_POST_COMMANDS | List of bash commands to run after the linter"
1404-
f"| {dump_as_json(linter.post_commands,'None')} |",
1441+
f"| {dump_as_json(linter.post_commands, 'None')} |",
14051442
]
14061443
add_in_config_schema_file(
14071444
[
@@ -2354,7 +2391,7 @@ def finalize_doc_build():
23542391
[![GitHub stars](https://img.shields.io/github/stars/oxsecurity/megalinter?cacheSeconds=3600&color=%23FD80CD)](https://github.com/oxsecurity/megalinter/stargazers/)
23552392
[![Dependents](https://img.shields.io/static/v1?label=Used%20by&message=2011&color=%23FD80CD&logo=slickpic)](https://github.com/oxsecurity/megalinter/network/dependents)
23562393
[![GitHub contributors](https://img.shields.io/github/contributors/oxsecurity/megalinter.svg?color=%23FD80CD)](https://github.com/oxsecurity/megalinter/graphs/contributors/)
2357-
[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square&color=%23FD80CD)](http://makeapullrequest.com)""", # noqa: E501
2394+
[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square&color=%23FD80CD)](http://makeapullrequest.com)""", # noqa: E501
23582395
)
23592396

23602397
# Remove TOC in target file
@@ -3085,7 +3122,7 @@ def update_workflow_linters(file_path, linters):
30853122
file_content = f.read()
30863123
file_content = re.sub(
30873124
r"(linter:\s+\[\s*)([^\[\]]*?)(\s*\])",
3088-
rf"\1{re.escape(linters).replace(chr(92),'').strip()}\3",
3125+
rf"\1{re.escape(linters).replace(chr(92), '').strip()}\3",
30893126
file_content,
30903127
)
30913128

‎Dockerfile

+35-1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,39 @@ FROM alpine/terragrunt:latest as terragrunt
4040
# Next FROM line commented because already managed by another linter
4141
# FROM alpine/terragrunt:latest as terragrunt
4242
FROM checkmarx/kics:alpine as kics
43+
FROM --platform=$BUILDPLATFORM alpine:3 AS cargo-build
44+
WORKDIR /cargo
45+
ENV HOME=/cargo
46+
USER 0
47+
RUN --mount=type=cache,target=/var/cache/apk,id=apk-${BUILDARCH},sharing=locked \
48+
apk add --update \
49+
gcc \
50+
rustup \
51+
bash \
52+
git \
53+
musl-dev \
54+
llvm \
55+
clang
56+
RUN chown 63425:63425 /cargo
57+
USER 63425
58+
ENV CC_aarch64_unknown_linux_musl=clang \
59+
AR_aarch64_unknown_linux_musl=llvm-ar \
60+
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld" \
61+
CC_x86_64_unknown_linux_musl=clang \
62+
AR_x86_64_unknown_linux_musl=llvm-ar \
63+
CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld"
64+
ARG TARGETARCH
65+
RUN rustup-init -y --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
66+
67+
RUN --mount=type=cache,id=cargo-${TARGETARCH},sharing=locked,target=/cargo/.cargo/registry/,uid=63425 \
68+
. /cargo/.cargo/env \
69+
&& cargo install shellcheck-sarif sarif-fmt --root /tmp --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
70+
71+
FROM scratch AS cargo
72+
COPY --link --from=cargo-build /tmp/bin/* /bin/
73+
RUN ["/bin/shellcheck-sarif", "--help"]
74+
RUN ["/bin/sarif-fmt", "--help"]
75+
4376
#FROM__END
4477

4578
##################
@@ -302,7 +335,7 @@ RUN echo 'gem: --no-document' >> ~/.gemrc && \
302335
#CARGO__START
303336
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable \
304337
&& export PATH="/root/.cargo/bin:${PATH}" \
305-
&& rustup component add clippy && cargo install --force --locked sarif-fmt shellcheck-sarif \
338+
&& rustup component add clippy \
306339
&& rm -rf /root/.cargo/registry /root/.cargo/git /root/.cache/sccache
307340
ENV PATH="/root/.cargo/bin:${PATH}"
308341
#CARGO__END
@@ -336,6 +369,7 @@ COPY --link --from=terragrunt /usr/local/bin/terragrunt /usr/bin/
336369
COPY --link --from=terragrunt /bin/terraform /usr/bin/
337370
COPY --link --from=kics /app/bin/kics /usr/bin/
338371
COPY --from=kics /app/bin/assets /opt/kics/assets/
372+
COPY --from=cargo /bin/* /usr/bin/
339373
#COPY__END
340374

341375
#############################################################################################

‎flavors/ci_light/Dockerfile

+35-5
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,39 @@ FROM mvdan/shfmt:latest-alpine as shfmt
1717
FROM hadolint/hadolint:v2.12.0-alpine as hadolint
1818
FROM mrtazz/checkmake:latest as checkmake
1919
FROM zricethezav/gitleaks:v8.16.1 as gitleaks
20+
FROM --platform=$BUILDPLATFORM alpine:3 AS cargo-build
21+
WORKDIR /cargo
22+
ENV HOME=/cargo
23+
USER 0
24+
RUN --mount=type=cache,target=/var/cache/apk,id=apk-${BUILDARCH},sharing=locked \
25+
apk add --update \
26+
gcc \
27+
rustup \
28+
bash \
29+
git \
30+
musl-dev \
31+
llvm \
32+
clang
33+
RUN chown 63425:63425 /cargo
34+
USER 63425
35+
ENV CC_aarch64_unknown_linux_musl=clang \
36+
AR_aarch64_unknown_linux_musl=llvm-ar \
37+
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld" \
38+
CC_x86_64_unknown_linux_musl=clang \
39+
AR_x86_64_unknown_linux_musl=llvm-ar \
40+
CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld"
41+
ARG TARGETARCH
42+
RUN rustup-init -y --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
43+
44+
RUN --mount=type=cache,id=cargo-${TARGETARCH},sharing=locked,target=/cargo/.cargo/registry/,uid=63425 \
45+
. /cargo/.cargo/env \
46+
&& cargo install shellcheck-sarif sarif-fmt --root /tmp --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
47+
48+
FROM scratch AS cargo
49+
COPY --link --from=cargo-build /tmp/bin/* /bin/
50+
RUN ["/bin/shellcheck-sarif", "--help"]
51+
RUN ["/bin/sarif-fmt", "--help"]
52+
2053
#FROM__END
2154

2255
##################
@@ -167,11 +200,7 @@ RUN echo 'gem: --no-document' >> ~/.gemrc && \
167200
#############################################################################################
168201

169202
#CARGO__START
170-
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable \
171-
&& export PATH="/root/.cargo/bin:${PATH}" \
172-
&& cargo install --force --locked sarif-fmt shellcheck-sarif \
173-
&& rm -rf /root/.cargo/registry /root/.cargo/git /root/.cache/sccache /root/.rustup
174-
ENV PATH="/root/.cargo/bin:${PATH}"
203+
175204
#CARGO__END
176205

177206
##############################
@@ -186,6 +215,7 @@ COPY --link --from=shfmt /bin/shfmt /usr/bin/
186215
COPY --link --from=hadolint /bin/hadolint /usr/bin/hadolint
187216
COPY --link --from=checkmake /checkmake /usr/bin/checkmake
188217
COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/
218+
COPY --from=cargo /bin/* /usr/bin/
189219
#COPY__END
190220

191221
#############################################################################################

‎flavors/cupcake/Dockerfile

+35-1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,39 @@ FROM alpine/terragrunt:latest as terragrunt
3636
# Next FROM line commented because already managed by another linter
3737
# FROM alpine/terragrunt:latest as terragrunt
3838
FROM checkmarx/kics:alpine as kics
39+
FROM --platform=$BUILDPLATFORM alpine:3 AS cargo-build
40+
WORKDIR /cargo
41+
ENV HOME=/cargo
42+
USER 0
43+
RUN --mount=type=cache,target=/var/cache/apk,id=apk-${BUILDARCH},sharing=locked \
44+
apk add --update \
45+
gcc \
46+
rustup \
47+
bash \
48+
git \
49+
musl-dev \
50+
llvm \
51+
clang
52+
RUN chown 63425:63425 /cargo
53+
USER 63425
54+
ENV CC_aarch64_unknown_linux_musl=clang \
55+
AR_aarch64_unknown_linux_musl=llvm-ar \
56+
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld" \
57+
CC_x86_64_unknown_linux_musl=clang \
58+
AR_x86_64_unknown_linux_musl=llvm-ar \
59+
CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld"
60+
ARG TARGETARCH
61+
RUN rustup-init -y --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
62+
63+
RUN --mount=type=cache,id=cargo-${TARGETARCH},sharing=locked,target=/cargo/.cargo/registry/,uid=63425 \
64+
. /cargo/.cargo/env \
65+
&& cargo install shellcheck-sarif sarif-fmt --root /tmp --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
66+
67+
FROM scratch AS cargo
68+
COPY --link --from=cargo-build /tmp/bin/* /bin/
69+
RUN ["/bin/shellcheck-sarif", "--help"]
70+
RUN ["/bin/sarif-fmt", "--help"]
71+
3972
#FROM__END
4073

4174
##################
@@ -262,7 +295,7 @@ RUN echo 'gem: --no-document' >> ~/.gemrc && \
262295
#CARGO__START
263296
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable \
264297
&& export PATH="/root/.cargo/bin:${PATH}" \
265-
&& rustup component add clippy && cargo install --force --locked sarif-fmt shellcheck-sarif \
298+
&& rustup component add clippy \
266299
&& rm -rf /root/.cargo/registry /root/.cargo/git /root/.cache/sccache
267300
ENV PATH="/root/.cargo/bin:${PATH}"
268301
#CARGO__END
@@ -293,6 +326,7 @@ COPY --link --from=terragrunt /usr/local/bin/terragrunt /usr/bin/
293326
COPY --link --from=terragrunt /bin/terraform /usr/bin/
294327
COPY --link --from=kics /app/bin/kics /usr/bin/
295328
COPY --from=kics /app/bin/assets /opt/kics/assets/
329+
COPY --from=cargo /bin/* /usr/bin/
296330
#COPY__END
297331

298332
#############################################################################################

‎flavors/documentation/Dockerfile

+35-5
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,39 @@ FROM mstruebing/editorconfig-checker:2.7.0 as editorconfig-checker
2424
FROM mrtazz/checkmake:latest as checkmake
2525
FROM yoheimuta/protolint:latest as protolint
2626
FROM zricethezav/gitleaks:v8.16.1 as gitleaks
27+
FROM --platform=$BUILDPLATFORM alpine:3 AS cargo-build
28+
WORKDIR /cargo
29+
ENV HOME=/cargo
30+
USER 0
31+
RUN --mount=type=cache,target=/var/cache/apk,id=apk-${BUILDARCH},sharing=locked \
32+
apk add --update \
33+
gcc \
34+
rustup \
35+
bash \
36+
git \
37+
musl-dev \
38+
llvm \
39+
clang
40+
RUN chown 63425:63425 /cargo
41+
USER 63425
42+
ENV CC_aarch64_unknown_linux_musl=clang \
43+
AR_aarch64_unknown_linux_musl=llvm-ar \
44+
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld" \
45+
CC_x86_64_unknown_linux_musl=clang \
46+
AR_x86_64_unknown_linux_musl=llvm-ar \
47+
CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld"
48+
ARG TARGETARCH
49+
RUN rustup-init -y --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
50+
51+
RUN --mount=type=cache,id=cargo-${TARGETARCH},sharing=locked,target=/cargo/.cargo/registry/,uid=63425 \
52+
. /cargo/.cargo/env \
53+
&& cargo install shellcheck-sarif sarif-fmt --root /tmp --target $([[ "${TARGETARCH}" == "amd64" ]] && echo "x86_64-unknown-linux-musl" || echo "aarch64-unknown-linux-musl")
54+
55+
FROM scratch AS cargo
56+
COPY --link --from=cargo-build /tmp/bin/* /bin/
57+
RUN ["/bin/shellcheck-sarif", "--help"]
58+
RUN ["/bin/sarif-fmt", "--help"]
59+
2760
#FROM__END
2861

2962
##################
@@ -199,11 +232,7 @@ RUN echo 'gem: --no-document' >> ~/.gemrc && \
199232
#############################################################################################
200233

201234
#CARGO__START
202-
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable \
203-
&& export PATH="/root/.cargo/bin:${PATH}" \
204-
&& cargo install --force --locked sarif-fmt shellcheck-sarif \
205-
&& rm -rf /root/.cargo/registry /root/.cargo/git /root/.cache/sccache /root/.rustup
206-
ENV PATH="/root/.cargo/bin:${PATH}"
235+
207236
#CARGO__END
208237

209238
##############################
@@ -225,6 +254,7 @@ COPY --link --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checke
225254
COPY --link --from=checkmake /checkmake /usr/bin/checkmake
226255
COPY --link --from=protolint /usr/local/bin/protolint /usr/bin/
227256
COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/
257+
COPY --from=cargo /bin/* /usr/bin/
228258
#COPY__END
229259

230260
#############################################################################################

0 commit comments

Comments
 (0)