Skip to content

Commit 1c5690c

Browse files
fschlimbfschlimb
authored
Sdl (#86)
* adding cmake to conda env * new pre-commit hooks, including bandit and isort * using recommended compiler/linker args, removing warnings * adding Security.md
1 parent 1f3634c commit 1c5690c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+631
-194
lines changed

‎.bandit.yml

+400
Large diffs are not rendered by default.

‎.pre-commit-config.yaml

+23-2
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,19 @@ repos:
44
- repo: https://github.com/pre-commit/pre-commit-hooks
55
rev: v4.5.0
66
hooks:
7-
- id: end-of-file-fixer
8-
- id: trailing-whitespace
7+
- id: check-ast
8+
- id: check-builtin-literals
9+
- id: check-case-conflict
10+
- id: check-executables-have-shebangs
11+
- id: check-merge-conflict
12+
- id: check-shebang-scripts-are-executable
13+
- id: check-toml
14+
- id: debug-statements
15+
- id: destroyed-symlinks
16+
- id: end-of-file-fixer
17+
- id: fix-byte-order-marker
18+
- id: mixed-line-ending
19+
- id: trailing-whitespace
920
- repo: https://github.com/pocc/pre-commit-hooks
1021
rev: v1.3.5
1122
hooks:
@@ -16,3 +27,13 @@ repos:
1627
hooks:
1728
- id: black
1829
language_version: python3
30+
- repo: https://github.com/PyCQA/bandit
31+
rev: '1.7.7'
32+
hooks:
33+
- id: bandit
34+
args: ["-c", ".bandit.yml"]
35+
- repo: https://github.com/pycqa/isort
36+
rev: 5.13.2
37+
hooks:
38+
- id: isort
39+
name: isort (python)

‎CMakeLists.txt

+12-3
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,18 @@ list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
6868
#include(AddIMEX)
6969

7070
# macro for mlir root directory
71-
add_compile_definitions(CMAKE_MLIR_ROOT="${MLIR_ROOT}" CMAKE_IMEX_ROOT="${IMEX_ROOT}")
71+
add_compile_definitions(CMAKE_MLIR_ROOT="${MLIR_ROOT}" CMAKE_IMEX_ROOT="${IMEX_ROOT}" FORTIFY_SOURCE=2 USE_MKL=1)
72+
add_compile_options(
73+
"-ftemplate-backtrace-limit=0"
74+
"-flarge-source-files"
75+
"-Wall" "-Wformat" "-fPIC" "-Wno-comment"
76+
"$<$<CONFIG:DEBUG>:-fstack-protector>"
77+
"$<$<CONFIG:RELEASE>:-flto>"
78+
"$<$<CONFIG:RELEASE>:-z>"
79+
"$<$<CONFIG:RELEASE>:noexecstack>"
80+
"$<$<CONFIG:RELEASE>:-fstack-protector-strong>"
81+
)
82+
add_link_options("-pie" "$<$<CONFIG:RELEASE>:-Wl,-z,relro,-z,now>")
7283

7384
#find_package(OpenMP)
7485

@@ -127,8 +138,6 @@ pybind11_add_module(_sharpy MODULE ${SHARPYSrcs} ${Hpps})
127138
add_library(idtr SHARED ${IDTRSrcs} ${Hpps})
128139
set(AllTargets _sharpy idtr)
129140

130-
add_compile_definitions(USE_MKL=1)
131-
add_compile_options("-ftemplate-backtrace-limit=0")
132141
include_directories(
133142
${PROJECT_SOURCE_DIR}/src/include
134143
${PROJECT_SOURCE_DIR}/third_party/bitsery/include

‎Security.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Security Policy
2+
Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation.
3+
4+
## Reporting a Vulnerability
5+
Please report any security vulnerabilities in this project [utilizing the guidelines here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html).

‎conda-env.txt

+13-1
Original file line numberDiff line numberDiff line change
@@ -20,21 +20,32 @@ https://conda.anaconda.org/intel/linux-64/tbb-devel-2021.11.0-intel_49513.tar.bz
2020
https://conda.anaconda.org/intel/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2
2121
https://conda.anaconda.org/intel/linux-64/libgcc-ng-13.2.0-h807b86a_2.conda
2222
https://conda.anaconda.org/intel/linux-64/bzip2-1.0.8-hb9a14ef_9.tar.bz2
23+
https://conda.anaconda.org/intel/linux-64/c-ares-1.21.0-hd590300_0.conda
24+
https://repo.anaconda.com/pkgs/main/linux-64/expat-2.5.0-h6a678d5_0.conda
25+
https://conda.anaconda.org/intel/linux-64/libev-4.33-h7f8727e_1.tar.bz2
2326
https://conda.anaconda.org/intel/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2
2427
https://conda.anaconda.org/intel/linux-64/libnsl-2.0.1-hd590300_0.conda
2528
https://conda.anaconda.org/intel/linux-64/libuuid-2.38.1-h0b41bf4_0.conda
29+
https://repo.anaconda.com/pkgs/main/linux-64/libuv-1.44.2-h5eee18b_0.conda
2630
https://conda.anaconda.org/intel/linux-64/libzlib-1.2.13-hd590300_5.conda
2731
https://conda.anaconda.org/intel/linux-64/ncurses-6.4-h59595ed_2.conda
2832
https://repo.anaconda.com/pkgs/main/linux-64/ninja-base-1.10.2-hd09550d_5.conda
2933
https://conda.anaconda.org/intel/linux-64/openssl-3.1.4-hd590300_0.conda
30-
https://conda.anaconda.org/intel/linux-64/xz-5.2.8-h5eee18b_0.tar.bz2
34+
https://repo.anaconda.com/pkgs/main/linux-64/xz-5.4.5-h5eee18b_0.conda
35+
https://repo.anaconda.com/pkgs/main/linux-64/libedit-3.1.20230828-h5eee18b_0.conda
3136
https://conda.anaconda.org/intel/linux-64/libsqlite-3.44.0-h2797004_0.conda
37+
https://repo.anaconda.com/pkgs/main/linux-64/libssh2-1.10.0-hdbd6064_2.conda
3238
https://conda.anaconda.org/intel/linux-64/readline-8.2-h8228510_1.conda
39+
https://repo.anaconda.com/pkgs/main/linux-64/rhash-1.4.3-hdbd6064_0.conda
3340
https://conda.anaconda.org/intel/linux-64/tk-8.6.13-noxft_h4845f30_101.conda
3441
https://conda.anaconda.org/intel/linux-64/zlib-1.2.13-hd590300_5.conda
42+
https://conda.anaconda.org/intel/linux-64/zstd-1.5.5-hfc55251_0.conda
43+
https://repo.anaconda.com/pkgs/main/linux-64/krb5-1.20.1-h143b758_1.conda
44+
https://repo.anaconda.com/pkgs/main/linux-64/libnghttp2-1.57.0-h2d74bed_0.conda
3545
https://conda.anaconda.org/intel/linux-64/python-3.10.13-h4499717_0.tar.bz2
3646
https://repo.anaconda.com/pkgs/main/linux-64/exceptiongroup-1.2.0-py310h06a4308_0.conda
3747
https://repo.anaconda.com/pkgs/main/noarch/iniconfig-1.1.1-pyhd3eb1b0_0.tar.bz2
48+
https://repo.anaconda.com/pkgs/main/linux-64/libcurl-8.5.0-h251f7ec_0.conda
3849
https://conda.anaconda.org/intel/linux-64/mpi4py-3.1.4-py310h618b5fa_0.tar.bz2
3950
https://repo.anaconda.com/pkgs/main/linux-64/ninja-1.10.2-h06a4308_5.conda
4051
https://conda.anaconda.org/intel/noarch/packaging-23.2-pyhd8ed1ab_0.conda
@@ -45,6 +56,7 @@ https://conda.anaconda.org/intel/noarch/six-1.16.0-pyhd3eb1b0_1.tar.bz2
4556
https://conda.anaconda.org/intel/linux-64/tbb4py-2021.11.0-py310_intel_49513.tar.bz2
4657
https://repo.anaconda.com/pkgs/main/linux-64/tomli-2.0.1-py310h06a4308_0.conda
4758
https://conda.anaconda.org/intel/noarch/wheel-0.41.3-pyhd8ed1ab_0.conda
59+
https://repo.anaconda.com/pkgs/main/linux-64/cmake-3.26.4-h96355d8_0.conda
4860
https://conda.anaconda.org/intel/linux-64/mkl-service-2.4.0-py310h3539a15_41.tar.bz2
4961
https://conda.anaconda.org/intel/noarch/pip-23.3.1-pyhd8ed1ab_0.conda
5062
https://repo.anaconda.com/pkgs/main/linux-64/pybind11-2.10.4-py310hdb19cb5_0.conda

‎examples/black_scholes.py

+4-3
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,10 @@
2222
python black_scholes.py -p M16Gb -b numpy
2323
2424
"""
25-
import numpy
26-
import time as time_mod
2725
import argparse
26+
import time as time_mod
27+
28+
import numpy
2829

2930
try:
3031
import mpi4py
@@ -150,7 +151,7 @@ def black_scholes(np, erf, nopt, price, strike, t, rate, volatility, call, put):
150151
def run(nopt, backend, iterations, datatype):
151152
if backend == "sharpy":
152153
import sharpy as np
153-
from sharpy import init, fini, sync
154+
from sharpy import fini, init, sync
154155

155156
init(False)
156157
erf = np.erf

‎examples/shallow_water.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,21 @@
2323
python shallow_water.py -b numpy ...
2424
2525
"""
26+
import argparse
2627
import math
27-
import numpy
28+
import os
2829
import time as time_mod
29-
import argparse
3030

31-
import os
31+
import numpy
3232

3333
device = os.getenv("SHARPY_USE_GPU", "")
3434

3535

3636
def run(n, backend, datatype, benchmark_mode):
3737
if backend == "sharpy":
3838
import sharpy as np
39+
from sharpy import fini, init, sync
3940
from sharpy.numpy import fromfunction
40-
from sharpy import init, fini, sync
4141

4242
all_axes = [0, 1]
4343
init(False)

‎examples/stencil-2d.py

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
#!/usr/bin/env python3
2-
#
31
# Copyright (c) 2015, Intel Corporation
42
#
53
# Redistribution and use in source and binary forms, with or without
@@ -55,7 +53,8 @@
5553
#
5654
# *******************************************************************
5755

58-
import sys, os
56+
import os
57+
import sys
5958

6059
print(
6160
"Python version = ", str(sys.version_info.major) + "." + str(sys.version_info.minor)

‎examples/wave_equation.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -23,21 +23,21 @@
2323
python wave_equation.py -b numpy ...
2424
2525
"""
26+
import argparse
2627
import math
27-
import numpy
28+
import os
2829
import time as time_mod
29-
import argparse
3030

31-
import os
31+
import numpy
3232

3333
device = os.getenv("SHARPY_USE_GPU", "")
3434

3535

3636
def run(n, backend, datatype, benchmark_mode):
3737
if backend == "sharpy":
3838
import sharpy as np
39+
from sharpy import fini, init, sync
3940
from sharpy.numpy import fromfunction
40-
from sharpy import init, fini, sync
4141

4242
all_axes = [0, 1]
4343
init(False)

‎scripts/code_gen.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
import sys, os
1+
import os
2+
import sys
23

34
sys.path.append(os.path.abspath(os.path.dirname(sys.argv[1])))
45
exec(f"import {os.path.basename(sys.argv[1]).split('.')[0]} as api")

‎setup.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import os
22
import pathlib
3-
from setuptools import setup, Extension
3+
4+
from setuptools import Extension, setup
45
from setuptools.command.build_ext import build_ext as build_ext_orig
56

67

‎sharpy/__init__.py

+18-20
Original file line numberDiff line numberDiff line change
@@ -14,30 +14,28 @@
1414
# are simply forwarded as-is.
1515

1616
_bool = bool
17+
from importlib import import_module
18+
from os import getenv
1719
from typing import Any
18-
from . import _sharpy as _csp
19-
from ._sharpy import (
20-
FLOAT64 as float64,
21-
FLOAT32 as float32,
22-
INT64 as int64,
23-
INT32 as int32,
24-
INT16 as int16,
25-
INT8 as int8,
26-
UINT64 as uint64,
27-
UINT32 as uint32,
28-
UINT16 as uint16,
29-
UINT8 as uint8,
30-
BOOL as bool,
31-
init as _init,
32-
fini,
33-
sync,
34-
)
3520

36-
from .ndarray import ndarray
37-
from os import getenv
38-
from importlib import import_module
21+
from . import _sharpy as _csp
3922
from . import array_api as api
4023
from . import spmd
24+
from ._sharpy import BOOL as bool
25+
from ._sharpy import FLOAT32 as float32
26+
from ._sharpy import FLOAT64 as float64
27+
from ._sharpy import INT8 as int8
28+
from ._sharpy import INT16 as int16
29+
from ._sharpy import INT32 as int32
30+
from ._sharpy import INT64 as int64
31+
from ._sharpy import UINT8 as uint8
32+
from ._sharpy import UINT16 as uint16
33+
from ._sharpy import UINT32 as uint32
34+
from ._sharpy import UINT64 as uint64
35+
from ._sharpy import fini
36+
from ._sharpy import init as _init
37+
from ._sharpy import sync
38+
from .ndarray import ndarray
4139

4240
_sharpy_cw = _bool(int(getenv("SHARPY_CW", False)))
4341

‎sharpy/numpy/random.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
from .. import _sharpy as _csp
21
from .. import __impl
2+
from .. import _sharpy as _csp
33
from ..ndarray import ndarray
44

55

‎sharpy/spmd.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ def get_locals(obj):
1212

1313
def from_locals(objs):
1414
arg = objs if isinstance(objs, (list, tuple)) else [objs]
15-
return ndarray(_csp._from_locals(arg))
15+
return ndarray.ndarray(_csp._from_locals(arg))
1616

1717

1818
def gather(obj, root=_csp._Ranks._REPLICATED):

‎sharpy/torch/__init__.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
from .. import _sharpy as _csp
21
import ndarray
32

3+
from .. import _sharpy as _csp
4+
45

56
def manual_seed(s=None):
67
ndarray.__impl.manual_seed(s + _csp.myrank() if s else _csp.myrank())

‎src/CollComm.cpp

+2-3
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,7 @@ void bufferize(NDArray::ptr_type a_ptr, void *outPtr) {
2222

2323
// @param outPtr touched only if on root and/or root==REPLICATED or if not
2424
// distributed
25-
void gather_array(NDArray::ptr_type a_ptr, rank_type root,
26-
void *outPtr) {
25+
void gather_array(NDArray::ptr_type a_ptr, rank_type root, void *outPtr) {
2726
auto trscvr = a_ptr->transceiver();
2827

2928
if (!trscvr || a_ptr->owner() == REPLICATED) {
@@ -58,7 +57,7 @@ void gather_array(NDArray::ptr_type a_ptr, rank_type root,
5857

5958
// compute each pranks local contribution
6059
int64_t curr = 0;
61-
for (auto i = 0; i < nranks; ++i) {
60+
for (auto i = 0ul; i < nranks; ++i) {
6261
assert(szsAndOffs[i * 2] * myTileSz == curr);
6362
displacements[i] = curr;
6463
counts[i] = szsAndOffs[i * 2 + 1] * myTileSz;

‎src/Creator.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ struct DeferredFull : public Deferred {
6262
bool generate_mlir(::mlir::OpBuilder &builder, const ::mlir::Location &loc,
6363
jit::DepManager &dm) override {
6464
::mlir::SmallVector<::mlir::Value> shp(rank());
65-
for (auto i = 0; i < rank(); ++i) {
65+
for (auto i = 0ul; i < rank(); ++i) {
6666
shp[i] = ::imex::createIndex(loc, builder, shape()[i]);
6767
}
6868

‎src/EWBinOp.cpp

-1
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,6 @@ struct DeferredEWBinOp : public Deferred {
121121
auto bv = dm.getDependent(builder, Registry::get(_b));
122122

123123
auto aTyp = av.getType().cast<::imex::ndarray::NDArrayType>();
124-
auto bTyp = bv.getType().cast<::imex::ndarray::NDArrayType>();
125124
auto outElemType =
126125
::imex::ndarray::toMLIR(builder, SHARPY::jit::getPTDType(_dtype));
127126
auto outTyp = aTyp.cloneWith(shape(), outElemType);

‎src/LinAlgOp.cpp

+5-4
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@
66

77
#include <mpi.h>
88
//#include <mkl.h>
9-
#include "sharpy/NDArray.hpp"
109
#include "sharpy/Factory.hpp"
1110
#include "sharpy/LinAlgOp.hpp"
11+
#include "sharpy/NDArray.hpp"
1212
#include "sharpy/TypeDispatch.hpp"
1313

1414
namespace SHARPY {
@@ -125,8 +125,8 @@ struct DeferredLinAlgOp : public Deferred {
125125
int _axis;
126126

127127
DeferredLinAlgOp() = default;
128-
DeferredLinAlgOp(const array_i::future_type &a,
129-
const array_i::future_type &b, int axis)
128+
DeferredLinAlgOp(const array_i::future_type &a, const array_i::future_type &b,
129+
int axis)
130130
: _a(a.guid()), _b(b.guid()), _axis(axis) {}
131131

132132
void run() {
@@ -144,7 +144,8 @@ struct DeferredLinAlgOp : public Deferred {
144144
}
145145
};
146146

147-
FutureArray *LinAlgOp::vecdot(const FutureArray &a, const FutureArray &b, int axis) {
147+
FutureArray *LinAlgOp::vecdot(const FutureArray &a, const FutureArray &b,
148+
int axis) {
148149
return new FutureArray(defer<DeferredLinAlgOp>(a.get(), b.get(), axis));
149150
}
150151

‎src/MPIMediator.cpp

+3-2
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ void MPIMediator::listen() {
141141
return;
142142

143143
constexpr int BSZ = 256;
144-
MPI_Request request_in = MPI_REQUEST_NULL, request_out = MPI_REQUEST_NULL;
144+
MPI_Request request_in = MPI_REQUEST_NULL;
145145
Buffer rbuff;
146146
// Issue async recv request
147147
Buffer buff(BSZ);
@@ -151,7 +151,6 @@ void MPIMediator::listen() {
151151
MPI_Status status;
152152
// Wait for any request
153153
MPI_Wait(&request_in, &status);
154-
rank_type requester = status.MPI_SOURCE;
155154
int cnt;
156155
MPI_Get_count(&status, MPI_CHAR, &cnt);
157156
buff.resize(cnt);
@@ -169,6 +168,8 @@ void MPIMediator::listen() {
169168
break;
170169
}
171170
#if 0
171+
rank_type requester = status.MPI_SOURCE;
172+
MPI_Request request_out = MPI_REQUEST_NULL;
172173
case PULL_TAG: {
173174
uint64_t id;
174175
ser.value8b(id);

‎src/MPITransceiver.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ MPITransceiver::MPITransceiver(bool is_cw)
7373
}
7474

7575
const char *clientArgs[args.size() + 1];
76-
for (int i = 0; i < args.size(); ++i)
76+
for (auto i = 0ul; i < args.size(); ++i)
7777
clientArgs[i] = args[i].c_str();
7878
clientArgs[args.size()] = nullptr;
7979

0 commit comments

Comments
 (0)