Skip to content

Commit 6120d79

Browse files
authored
build: Add mz-build-tools crate (#26632)
This PR introduces a new `mz-build-tools` crate which is designed to expose any external tools required in build scripts. Currently the only such tool is `protoc`. ### Motivation This is a lead up to building Materialize with Bazel. Today we depend on `protobuf-src` which builds `protoc` for us in its build script, and then returns the path of the compiled binary. For Bazel though, we don't want to depend on `protobuf-src` and instead let Bazel build `protoc`. ### Checklist - [ ] This PR has adequate test coverage / QA involvement has been duly considered. ([trigger-ci for additional test/nightly runs](https://trigger-ci.dev.materialize.com/)) - [ ] This PR has an associated up-to-date [design doc](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/README.md), is a design doc ([template](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/00000000_template.md)), or is sufficiently small to not require a design. <!-- Reference the design in the description. --> - [ ] If this PR evolves [an existing `$T ⇔ Proto$T` mapping](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/command-and-response-binary-encoding.md) (possibly in a backwards-incompatible way), then it is tagged with a `T-proto` label. - [ ] If this PR will require changes to cloud orchestration or tests, there is a companion cloud PR to account for those changes that is tagged with the release-blocker label ([example](MaterializeInc/cloud#5021)). <!-- Ask in #team-cloud on Slack if you need help preparing the cloud PR. --> - [x] This PR includes the following [user-facing behavior changes](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/guide-changes.md#what-changes-require-a-release-note): - N/a
1 parent a25a115 commit 6120d79

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+286
-114
lines changed

Cargo.lock

+41-31
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ members = [
99
"src/aws-secrets-controller",
1010
"src/balancerd",
1111
"src/build-info",
12+
"src/build-tools",
1213
"src/ccsr",
1314
"src/catalog",
1415
"src/catalog-debug",

doc/developer/command-and-response-binary-encoding.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ This step is only needed if `$T` is a complex type ([classes (c) or (d)](#type-c
130130

131131
```rust
132132
fn main() {
133-
env::set_var("PROTOC", protobuf_src::protoc());
133+
env::set_var("PROTOC", mz_build_tools::protoc());
134134
prost_build::Config::new()
135135
// list paths to external types used in the compiled files
136136
.extern_path(".mz_repr.adt.char", "::mz_repr::adt::char")

src/build-tools/Cargo.toml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
[package]
2+
name = "mz-build-tools"
3+
description = "Provides access to tools required in build scripts."
4+
version = "0.0.0"
5+
edition.workspace = true
6+
rust-version.workspace = true
7+
publish = false
8+
9+
[dependencies]
10+
cfg-if = "1.0.0"
11+
protobuf-src = { version = "1.1.0", optional = true }
12+
which = "4"
13+
workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = true }
14+
15+
[lints]
16+
workspace = true
17+
18+
[features]
19+
default = ["protobuf-src", "workspace-hack"]
20+
bazel = []
21+
22+
[package.metadata.cargo-udeps.ignore]
23+
normal = ["workspace-hack"]

src/build-tools/src/lib.rs

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Copyright Materialize, Inc. and contributors. All rights reserved.
2+
//
3+
// Use of this software is governed by the Business Source License
4+
// included in the LICENSE file.
5+
//
6+
// As of the Change Date specified in that file, in accordance with
7+
// the Business Source License, use of this software will be governed
8+
// by the Apache License, Version 2.0.
9+
10+
//! Provides access to tools required in build scripts.
11+
//!
12+
//! For example, many crates have a build script that depends on the Protobuf
13+
//! compiler, `protoc`. If we're building with Cargo we'll bootstrap `protoc`
14+
//! by compiling it with [`protobuf-src`], but if we're building with Bazel
15+
//! then we'll use the version of `protoc` included in the runfiles.
16+
17+
use cfg_if::cfg_if;
18+
use std::path::PathBuf;
19+
20+
// Note: This crate's BUILD.bazel compiles with the rustc flag `--cfg=bazel`.
21+
22+
// Runfiles are a Bazel concept, they're a way to provide files at execution
23+
// time. This dependency is provided only by the Bazel build.
24+
#[cfg(bazel)]
25+
extern crate runfiles;
26+
27+
/// Returns the path to `protoc`.
28+
///
29+
/// Looks for `protoc` in the following places:
30+
///
31+
/// * Bazel runfiles, if we're building with Bazel.
32+
/// * Bootstraps `protoc` via protobuf-src, if default features are enabled.
33+
/// * `PROTOC` environment variable, if it's set.
34+
/// * The system's `$PATH`, via [`which`].
35+
///
36+
/// If `protoc` can't be found then this function will panic.
37+
pub fn protoc() -> PathBuf {
38+
cfg_if! {
39+
if #[cfg(bazel)] {
40+
let r = runfiles::Runfiles::create().unwrap();
41+
r.rlocation("protobuf/protoc")
42+
} else if #[cfg(feature = "protobuf-src")] {
43+
protobuf_src::protoc()
44+
} else {
45+
// If we're not building with Bazel, nor have the `protobuf-src`
46+
// feature specified, then try using the system's `protoc`.
47+
match std::option_env!("PROTOC") {
48+
Some(path) => PathBuf::from(path),
49+
None => which::which("protoc").expect("protoc to exist on system"),
50+
}
51+
}
52+
}
53+
}
54+
55+
/// Returns the path to the protobuf includes directory.
56+
///
57+
/// Note: this is primarily used to include "well known types".
58+
pub fn protoc_include() -> PathBuf {
59+
cfg_if! {
60+
if #[cfg(bazel)] {
61+
let r = runfiles::Runfiles::create().unwrap();
62+
r.rlocation("protobuf/src")
63+
} else if #[cfg(feature = "protobuf-src")] {
64+
protobuf_src::include()
65+
} else {
66+
let path = std::option_env!("PROTOC_INCLUDE").unwrap_or_default();
67+
PathBuf::from(path)
68+
}
69+
}
70+
}

src/catalog/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -73,10 +73,13 @@ tokio-postgres = { version = "0.7.8" }
7373
[build-dependencies]
7474
anyhow = "1.0.66"
7575
md-5 = "0.10.5"
76-
protobuf-src = "1.1.0"
76+
mz-build-tools = { path = "../build-tools", default-features = false }
7777
prost-build = "0.11.9"
7878
serde = "1.0.152"
7979
serde_json = "1.0.89"
8080

81+
[features]
82+
default = ["mz-build-tools/default"]
83+
8184
[package.metadata.cargo-udeps.ignore]
8285
normal = ["workspace-hack"]

src/catalog/build.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ const PROTO_HASHES: &str = "protos/hashes.json";
3131
fn main() -> anyhow::Result<()> {
3232
println!("cargo:rerun-if-changed={PROTO_DIRECTORY}");
3333

34-
env::set_var("PROTOC", protobuf_src::protoc());
34+
env::set_var("PROTOC", mz_build_tools::protoc());
3535

3636
// Read in the persisted hashes from disk.
3737
let hashes = fs::File::open(PROTO_HASHES).context("opening proto hashes")?;

src/ccsr/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,11 @@ tokio = { version = "1.32.0", features = ["macros"] }
3333
tracing = "0.1.37"
3434

3535
[build-dependencies]
36+
mz-build-tools = { path = "../build-tools", default-features = false }
3637
prost-build = "0.11.2"
37-
protobuf-src = "1.1.0"
38+
39+
[features]
40+
default = ["mz-build-tools/default"]
3841

3942
[package.metadata.cargo-udeps.ignore]
4043
normal = ["workspace-hack"]

src/cluster-client/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,12 @@ uuid = { version = "1.7.0", features = ["serde", "v4"] }
3535
workspace-hack = { version = "0.0.0", path = "../workspace-hack" }
3636

3737
[build-dependencies]
38+
mz-build-tools = { path = "../build-tools", default-features = false }
3839
prost-build = "0.11.2"
39-
protobuf-src = "1.1.0"
4040
tonic-build = "0.9.2"
4141

42+
[features]
43+
default = ["mz-build-tools/default"]
44+
4245
[package.metadata.cargo-udeps.ignore]
4346
normal = ["workspace-hack"]

src/cluster-client/build.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
use std::env;
1111

1212
fn main() {
13-
env::set_var("PROTOC", protobuf_src::protoc());
13+
env::set_var("PROTOC", mz_build_tools::protoc());
1414

1515
let mut config = prost_build::Config::new();
1616
config.btree_map(["."]);

src/compute-client/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,12 @@ uuid = { version = "1.7.0", features = ["serde", "v4"] }
5555
workspace-hack = { version = "0.0.0", path = "../workspace-hack" }
5656

5757
[build-dependencies]
58+
mz-build-tools = { path = "../build-tools", default-features = false }
5859
prost-build = "0.11.2"
59-
protobuf-src = "1.1.0"
6060
tonic-build = "0.9.2"
6161

62+
[features]
63+
default = ["mz-build-tools/default"]
64+
6265
[package.metadata.cargo-udeps.ignore]
6366
normal = ["workspace-hack"]

src/compute-client/build.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
use std::env;
1111

1212
fn main() {
13-
env::set_var("PROTOC", protobuf_src::protoc());
13+
env::set_var("PROTOC", mz_build_tools::protoc());
1414

1515
let mut config = prost_build::Config::new();
1616
config

src/compute-types/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,12 @@ tracing = "0.1.37"
2828
workspace-hack = { version = "0.0.0", path = "../workspace-hack" }
2929

3030
[build-dependencies]
31+
mz-build-tools = { path = "../build-tools", default-features = false }
3132
prost-build = "0.11.2"
32-
protobuf-src = "1.1.0"
3333
tonic-build = "0.9.2"
3434

35+
[features]
36+
default = ["mz-build-tools/default"]
37+
3538
[package.metadata.cargo-udeps.ignore]
3639
normal = ["workspace-hack"]

src/compute-types/build.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
use std::env;
1111

1212
fn main() {
13-
env::set_var("PROTOC", protobuf_src::protoc());
13+
env::set_var("PROTOC", mz_build_tools::protoc());
1414

1515
let mut config = prost_build::Config::new();
1616
config

src/dyncfg/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,11 @@ tracing = "0.1.37"
2222
workspace-hack = { version = "0.0.0", path = "../workspace-hack" }
2323

2424
[build-dependencies]
25+
mz-build-tools = { path = "../build-tools", default-features = false }
2526
prost-build = "0.11.2"
26-
protobuf-src = "1.1.0"
27+
28+
[features]
29+
default = ["mz-build-tools/default"]
2730

2831
[package.metadata.cargo-udeps.ignore]
2932
normal = ["workspace-hack"]

src/dyncfg/build.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
use std::env;
1111

1212
fn main() {
13-
env::set_var("PROTOC", protobuf_src::protoc());
13+
env::set_var("PROTOC", mz_build_tools::protoc());
1414

1515
prost_build::Config::new()
1616
.btree_map(["."])

src/dyncfgs/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,11 @@ mz-storage-types = { path = "../storage-types" }
2020
workspace-hack = { version = "0.0.0", path = "../workspace-hack" }
2121

2222
[build-dependencies]
23+
mz-build-tools = { path = "../build-tools", default-features = false }
2324
prost-build = "0.11.2"
24-
protobuf-src = "1.1.0"
25+
26+
[features]
27+
default = ["mz-build-tools/default"]
2528

2629
[package.metadata.cargo-udeps.ignore]
2730
normal = ["workspace-hack"]

src/expr/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,11 @@ mz-ore = { path = "../ore" }
7171
proc-macro2 = "1.0.60"
7272

7373
[build-dependencies]
74+
mz-build-tools = { path = "../build-tools", default-features = false }
7475
prost-build = "0.11.2"
75-
protobuf-src = "1.1.0"
76+
77+
[features]
78+
default = ["mz-build-tools/default"]
7679

7780
[package.metadata.cargo-udeps.ignore]
7881
normal = ["workspace-hack"]

src/expr/build.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
use std::env;
1111

1212
fn main() {
13-
env::set_var("PROTOC", protobuf_src::protoc());
13+
env::set_var("PROTOC", mz_build_tools::protoc());
1414

1515
prost_build::Config::new()
1616
.extern_path(".mz_pgtz", "::mz_pgtz")

src/fivetran-destination/Cargo.toml

+2-2
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,10 @@ workspace-hack = { version = "0.0.0", path = "../workspace-hack", optional = tru
4343
insta = "1.32"
4444

4545
[build-dependencies]
46+
mz-build-tools = { path = "../build-tools", default-features = false }
4647
prost-build = "0.11.2"
47-
protobuf-src = { version = "1.1.0", optional = true }
4848
reqwest = { version = "0.11.13", features = ["blocking", "native-tls-vendored"] }
4949
tonic-build = "0.9.2"
5050

5151
[features]
52-
default = ["protobuf-src"]
52+
default = ["mz-build-tools/default", "workspace-hack"]

src/fivetran-destination/build.rs

+1-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ fn main() {
2020

2121
// Build protobufs.
2222
{
23-
#[cfg(feature = "protobuf-src")]
24-
env::set_var("PROTOC", protobuf_src::protoc());
23+
env::set_var("PROTOC", mz_build_tools::protoc());
2524

2625
let mut config = prost_build::Config::new();
2726
config.btree_map(["."]);

src/interchange/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,11 @@ criterion = { version = "0.4.0", features = ["async_tokio"] }
4242
tokio = { version = "1.32.0", features = ["macros"] }
4343

4444
[build-dependencies]
45+
mz-build-tools = { path = "../build-tools", default-features = false }
4546
prost-build = "0.11.2"
46-
protobuf-src = "1.1.0"
47+
48+
[features]
49+
default = ["mz-build-tools/default"]
4750

4851
[package.metadata.cargo-udeps.ignore]
4952
normal = ["workspace-hack"]

src/interchange/build.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ use std::env;
1111
use std::path::PathBuf;
1212

1313
fn main() {
14-
env::set_var("PROTOC", protobuf_src::protoc());
14+
env::set_var("PROTOC", mz_build_tools::protoc());
1515

1616
let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap());
1717

src/kafka-util/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,11 @@ url = "2.3.1"
3333
workspace-hack = { version = "0.0.0", path = "../workspace-hack" }
3434

3535
[build-dependencies]
36+
mz-build-tools = { path = "../build-tools", default-features = false }
3637
prost-build = "0.11.2"
37-
protobuf-src = "1.1.0"
38+
39+
[features]
40+
default = ["mz-build-tools/default"]
3841

3942
[package.metadata.cargo-udeps.ignore]
4043
normal = ["workspace-hack"]

src/mysql-util/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,12 @@ uuid = { version = "1.7.0", features = ["v4"] }
3535
workspace-hack = { version = "0.0.0", path = "../workspace-hack" }
3636

3737
[build-dependencies]
38+
mz-build-tools = { path = "../build-tools", default-features = false }
3839
prost-build = "0.11.2"
39-
protobuf-src = "1.1.0"
4040
tonic-build = "0.9.2"
4141

42+
[features]
43+
default = ["mz-build-tools/default"]
44+
4245
[package.metadata.cargo-udeps.ignore]
4346
normal = ["workspace-hack"]

src/mysql-util/build.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
use std::env;
1111

1212
fn main() {
13-
env::set_var("PROTOC", protobuf_src::protoc());
13+
env::set_var("PROTOC", mz_build_tools::protoc());
1414

1515
let mut config = prost_build::Config::new();
1616
config.btree_map(["."]);

src/orchestrator/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,11 @@ serde = "1.0"
2222
workspace-hack = { version = "0.0.0", path = "../workspace-hack" }
2323

2424
[build-dependencies]
25-
protobuf-src = "1.1.0"
25+
mz-build-tools = { path = "../build-tools", default-features = false }
2626
tonic-build = "0.9.2"
2727

28+
[features]
29+
default = ["mz-build-tools/default"]
30+
2831
[package.metadata.cargo-udeps.ignore]
2932
normal = ["workspace-hack"]

src/persist-client/Cargo.toml

+4-1
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,12 @@ num_cpus = "1.14.0"
7575
tempfile = "3.8.1"
7676

7777
[build-dependencies]
78+
mz-build-tools = { path = "../build-tools", default-features = false }
7879
prost-build = "0.11.2"
79-
protobuf-src = "1.1.0"
8080
tonic-build = "0.9.2"
8181

82+
[features]
83+
default = ["mz-build-tools/default"]
84+
8285
[package.metadata.cargo-udeps.ignore]
8386
normal = ["workspace-hack"]

src/persist-client/build.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
use std::env;
1111

1212
fn main() {
13-
env::set_var("PROTOC", protobuf_src::protoc());
13+
env::set_var("PROTOC", mz_build_tools::protoc());
1414

1515
let mut config = prost_build::Config::new();
1616
config

0 commit comments

Comments
 (0)