Skip to content

Allow shorter drive listing in config.yaml #6024

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 27, 2024
Merged

Allow shorter drive listing in config.yaml #6024

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions ydb/core/blobstorage/nodewarden/distconf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ namespace NKikimr::NStorage {

template<typename T>
void EnumerateConfigDrives(const NKikimrBlobStorage::TStorageConfig& config, ui32 nodeId, T&& callback,
THashMap<ui32, const NKikimrBlobStorage::TNodeIdentifier*> *nodeMap = nullptr) {
THashMap<ui32, const NKikimrBlobStorage::TNodeIdentifier*> *nodeMap = nullptr, bool fillInPDiskConfig = false) {
if (!config.HasBlobStorageConfig()) {
return;
}
Expand Down Expand Up @@ -458,9 +458,30 @@ namespace NKikimr::NStorage {
const auto& node = *it->second;
if (const auto it = defineHostConfigMap.find(host.GetHostConfigId()); it != defineHostConfigMap.end()) {
const auto& hostConfig = *it->second;
auto processDrive = [&](const auto& drive) {
if (fillInPDiskConfig && !drive.HasPDiskConfig() && hostConfig.HasDefaultHostPDiskConfig()) {
NKikimrBlobStorage::THostConfigDrive temp;
temp.CopyFrom(drive);
temp.MutablePDiskConfig()->CopyFrom(hostConfig.GetDefaultHostPDiskConfig());
callback(node, temp);
} else {
callback(node, drive);
}
};
for (const auto& drive : hostConfig.GetDrive()) {
callback(node, drive);
processDrive(drive);
}
auto processTypedDrive = [&](const auto& field, NKikimrBlobStorage::EPDiskType type) {
for (const auto& path : field) {
NKikimrBlobStorage::THostConfigDrive drive;
drive.SetType(type);
drive.SetPath(path);
processDrive(drive);
}
};
processTypedDrive(hostConfig.GetRot(), NKikimrBlobStorage::EPDiskType::ROT);
processTypedDrive(hostConfig.GetSsd(), NKikimrBlobStorage::EPDiskType::SSD);
processTypedDrive(hostConfig.GetNvme(), NKikimrBlobStorage::EPDiskType::NVME);
}
}
}
Expand Down
5 changes: 5 additions & 0 deletions ydb/core/blobstorage/nodewarden/distconf_fsm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,11 @@ namespace NKikimr::NStorage {
invoke(disk);
}
}
for (const auto& item : res->GetProposedConfigs()) {
for (const auto& disk : item.GetDisks()) {
invoke(disk);
}
}
for (const auto& disk : res->GetNoMetadata()) {
invoke(disk);
}
Expand Down
61 changes: 20 additions & 41 deletions ydb/core/blobstorage/nodewarden/distconf_generate.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ namespace NKikimr::NStorage {
using TPDiskId = NBsController::TPDiskId;

NKikimrConfig::TBlobStorageConfig *bsConfig = config->MutableBlobStorageConfig();
const auto& settings = bsConfig->GetAutoconfigSettings();

// build node location map
THashMap<ui32, TNodeLocation> nodeLocations;
Expand Down Expand Up @@ -301,50 +300,30 @@ namespace NKikimr::NStorage {
}

// build host config map
THashMap<ui64, const NKikimrBlobStorage::TDefineHostConfig*> hostConfigs;
for (const auto& hc : settings.GetDefineHostConfig()) {
const bool inserted = hostConfigs.try_emplace(hc.GetHostConfigId(), &hc).second;
Y_ABORT_UNLESS(inserted);
}

// find all drives
const auto& defineBox = settings.GetDefineBox();
for (const auto& host : defineBox.GetHost()) {
const ui32 nodeId = host.GetEnforcedNodeId();
if (!nodeId) {
throw TExConfigError() << "EnforcedNodeId is not specified in DefineBox";
}

const auto it = hostConfigs.find(host.GetHostConfigId());
if (it == hostConfigs.end()) {
throw TExConfigError() << "no matching DefineHostConfig"
<< " HostConfigId# " << host.GetHostConfigId();
auto processDrive = [&](const auto& node, const auto& drive) {
const ui32 nodeId = node.GetNodeId();
if (pdiskLocations.contains(std::make_tuple(nodeId, drive.GetPath()))) {
return;
}
const auto& defineHostConfig = *it->second;

for (const auto& drive : defineHostConfig.GetDrive()) {
if (pdiskLocations.contains(std::make_tuple(nodeId, drive.GetPath()))) {
continue;
}
if (checkMatch(drive.GetType(), drive.GetSharedWithOs(), drive.GetReadCentric(), drive.GetKind())) {
const TPDiskId pdiskId(nodeId, ++maxPDiskId[nodeId]);
if (const auto [it, inserted] = pdisks.try_emplace(pdiskId); inserted) {
auto& r = it->second.Record;
r.SetNodeID(pdiskId.NodeId);
r.SetPDiskID(pdiskId.PDiskId);
r.SetPath(drive.GetPath());
r.SetPDiskGuid(RandomNumber<ui64>());
r.SetPDiskCategory(TPDiskCategory(static_cast<NPDisk::EDeviceType>(drive.GetType()),
drive.GetKind()).GetRaw());
if (drive.HasPDiskConfig()) {
r.MutablePDiskConfig()->CopyFrom(drive.GetPDiskConfig());
}
} else {
Y_ABORT("duplicate PDiskId");
if (checkMatch(drive.GetType(), drive.GetSharedWithOs(), drive.GetReadCentric(), drive.GetKind())) {
const TPDiskId pdiskId(nodeId, ++maxPDiskId[nodeId]);
if (const auto [it, inserted] = pdisks.try_emplace(pdiskId); inserted) {
auto& r = it->second.Record;
r.SetNodeID(pdiskId.NodeId);
r.SetPDiskID(pdiskId.PDiskId);
r.SetPath(drive.GetPath());
r.SetPDiskGuid(RandomNumber<ui64>());
r.SetPDiskCategory(TPDiskCategory(static_cast<NPDisk::EDeviceType>(drive.GetType()),
drive.GetKind()));
if (drive.HasPDiskConfig()) {
r.MutablePDiskConfig()->CopyFrom(drive.GetPDiskConfig());
}
} else {
Y_ABORT("duplicate PDiskId");
}
}
}
};
EnumerateConfigDrives(*config, 0, processDrive, nullptr, true);

// group mapper
NBsController::TGroupGeometryInfo geom(gtype.GetErasure(), geometry);
Expand Down
24 changes: 23 additions & 1 deletion ydb/core/mind/bscontroller/cmds_host_config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,31 @@ namespace NKikimr::NBsController {
}

Schema::HostConfigDrive::TKey::Type key(id, drive.GetPath());
config.Drives.emplace(std::move(key), std::move(driveInfo));
const auto [it, inserted] = config.Drives.emplace(std::move(key), std::move(driveInfo));
if (!inserted) {
throw TExError() << "duplicate path# " << drive.GetPath();
}
}

auto addDrives = [&](const auto& field, NKikimrBlobStorage::EPDiskType type) {
THostConfigInfo::TDriveInfo driveInfo;
driveInfo.Type = type;
driveInfo.SharedWithOs = false;
driveInfo.ReadCentric = false;
driveInfo.Kind = 0;
driveInfo.PDiskConfig = defaultPDiskConfig;

for (const auto& path : field) {
const auto [it, inserted] = config.Drives.emplace(Schema::HostConfigDrive::TKey::Type(id, path), driveInfo);
if (!inserted) {
throw TExError() << "duplicate path# " << path;
}
}
};
addDrives(cmd.GetRot(), NKikimrBlobStorage::EPDiskType::ROT);
addDrives(cmd.GetSsd(), NKikimrBlobStorage::EPDiskType::SSD);
addDrives(cmd.GetNvme(), NKikimrBlobStorage::EPDiskType::NVME);

auto &hostConfigs = HostConfigs.Unshare();
hostConfigs[id] = std::move(config);

Expand Down
5 changes: 5 additions & 0 deletions ydb/core/protos/blobstorage_config.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ message TDefineHostConfig {
// host-wide default configuration for every PDisk
NKikimrBlobStorage.TPDiskConfig DefaultHostPDiskConfig = 4;

// some syntactic sugar -- one drive type per path
repeated string Rot = 5;
repeated string Ssd = 6;
repeated string Nvme = 7;

// item's generation to prevent concurrent modification
uint64 ItemConfigGeneration = 100;
}
Expand Down
4 changes: 4 additions & 0 deletions ydb/library/yaml_config/protos/config.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,10 @@ message TExtendedDefineHostConfig {

optional NKikimrBlobStorage.TPDiskConfig DefaultHostPDiskConfig = 4 [(NMarkers.CopyTo) = "TDefineHostConfig"];

repeated string Rot = 5 [(NMarkers.CopyTo) = "TDefineHostConfig"];
repeated string Ssd = 6 [(NMarkers.CopyTo) = "TDefineHostConfig"];
repeated string Nvme = 7 [(NMarkers.CopyTo) = "TDefineHostConfig"];

optional uint64 ItemConfigGeneration = 100 [(NMarkers.CopyTo) = "TDefineHostConfig"];
}

Expand Down
Loading