updating number_of_records_epsilon to number_of_records_epsilon_proportion (in terms of the input privacy budget)

Author: rracanicci

packages/cli/src/main.rs

@@ -189,11 +189,11 @@ enum Command {
         sigma_proportions: Option<Vec<f64>>,
 
         #[structopt(
-            long = "number-of-records-epsilon",
-            help = "epsilon used to add noise to the protected number of records in the aggregated data (default is 0.1)",
+            long = "number-of-records-epsilon-proportion",
+            help = "proportion of epsilon used to add noise to the protected number of records in the aggregated data (default is 0.005)",
             requires = "dp"
         )]
-        number_of_records_epsilon: Option<f64>,
+        number_of_records_epsilon_proportion: Option<f64>,
 
         #[structopt(
             long = "aggregates-json",
@@ -398,7 +398,7 @@ fn main() {
                 noise_threshold_type,
                 noise_threshold_values,
                 sigma_proportions,
-                number_of_records_epsilon,
+                number_of_records_epsilon_proportion,
                 aggregates_json,
             } => {
                 let mut aggregator = Aggregator::new(data_block.clone());
@@ -429,7 +429,7 @@ fn main() {
                             sensitivities_percentile.unwrap(),
                             sensitivities_epsilon_proportion.unwrap(),
                             sigma_proportions,
-                            number_of_records_epsilon,
+                            number_of_records_epsilon_proportion,
                         ),
                         threshold,
                         &mut progress_reporter,

packages/core/src/dp/dp_parameters.rs

@@ -2,8 +2,9 @@
 use pyo3::prelude::*;
 use serde::{Deserialize, Serialize};
 
-/// Default epsilon used to add noise to the protected number of records in the aggregated data
-pub const DEFAULT_NUMBER_OF_RECORDS_EPSILON: f64 = 0.1;
+/// Default epsilon proportion used to add noise to the protected number of records
+/// in the aggregated data
+pub const DEFAULT_NUMBER_OF_RECORDS_EPSILON_PROPORTION: f64 = 0.005;
 
 /// Parameters for aggregate generation with differential privacy
 #[cfg_attr(feature = "pyo3", pyclass)]
@@ -26,9 +27,9 @@ pub struct DpParameters {
     /// (e.g. \[1.0, 2.0, 3.0\] means that `sigma_2 = 2.0 * sigma_1` and `sigma_3 = 3.0 * sigma_1`)
     /// - If `None` all the sigma values will be the same
     pub sigma_proportions: Option<Vec<f64>>,
-    /// Epsilon used to add noise to the protected number of records in the aggregated data
-    /// (default is 0.1)
-    pub number_of_records_epsilon: Option<f64>,
+    /// Proportion of epsilon used to add noise to the protected number of records in
+    /// the aggregated data (default is 0.005)
+    pub number_of_records_epsilon_proportion: Option<f64>,
 }
 
 #[cfg_attr(feature = "pyo3", pymethods)]
@@ -49,23 +50,23 @@ impl DpParameters {
     /// controls how the budget being split across combination lengths
     /// (e.g. \[1.0, 2.0, 3.0\] means that `sigma_2 = 2.0 * sigma_1` and `sigma_3 = 3.0 * sigma_1`)
     ///     - If `None` all the sigma values will be the same
-    /// * `number_of_records_epsilon` - Epsilon used to add noise to the protected number of records in the aggregated data
-    /// (default is 0.1)
+    /// * `number_of_records_epsilon_proportion` - Proportion of epsilon used to add noise to the protected number of records
+    /// in the aggregated data (default is 0.005)
     pub fn new(
         epsilon: f64,
         delta: f64,
         percentile_percentage: usize,
         percentile_epsilon_proportion: f64,
         sigma_proportions: Option<Vec<f64>>,
-        number_of_records_epsilon: Option<f64>,
+        number_of_records_epsilon_proportion: Option<f64>,
     ) -> Self {
         DpParameters {
             epsilon,
             delta,
             percentile_percentage,
             percentile_epsilon_proportion,
             sigma_proportions,
-            number_of_records_epsilon,
+            number_of_records_epsilon_proportion,
         }
     }
 
@@ -84,23 +85,23 @@ impl DpParameters {
     /// controls how the budget being split across combination lengths
     /// (e.g. \[1.0, 2.0, 3.0\] means that `sigma_2 = 2.0 * sigma_1` and `sigma_3 = 3.0 * sigma_1`)
     ///     - If `None` all the sigma values will be the same
-    /// * `number_of_records_epsilon` - Epsilon used to add noise to the protected number of records in the aggregated data
-    /// (default is 0.1)
+    /// * `number_of_records_epsilon_proportion` - Proportion of epsilon used to add noise to the protected number of records
+    /// in the aggregated data (default is 0.005)
     pub fn new(
         epsilon: f64,
         delta: f64,
         percentile_percentage: usize,
         percentile_epsilon_proportion: f64,
         sigma_proportions: Option<Vec<f64>>,
-        number_of_records_epsilon: Option<f64>,
+        number_of_records_epsilon_proportion: Option<f64>,
     ) -> Self {
         DpParameters {
             epsilon,
             delta,
             percentile_percentage,
             percentile_epsilon_proportion,
             sigma_proportions,
-            number_of_records_epsilon,
+            number_of_records_epsilon_proportion,
         }
     }
 }

packages/core/src/dp/noise_aggregator.rs

@@ -1,6 +1,6 @@
 use super::{
     CombinationsByRecord, DpParameters, DpPercentile, NoisyCountThreshold,
-    DEFAULT_NUMBER_OF_RECORDS_EPSILON,
+    DEFAULT_NUMBER_OF_RECORDS_EPSILON_PROPORTION,
 };
 use fnv::FnvHashSet;
 use itertools::Itertools;
@@ -44,13 +44,14 @@ pub struct NoiseAggregator {
 
 impl NoiseAggregator {
     #[inline]
-    fn calc_percentile_epsilon_and_sigma_by_len(
+    fn calc_percentile_epsilon_number_of_records_epsilon_and_sigma_by_len(
         reporting_length: usize,
         epsilon: f64,
         delta: f64,
         percentile_epsilon_proportion: f64,
+        number_of_records_proportion: f64,
         sigma_proportions: &Option<Vec<f64>>,
-    ) -> (f64, Vec<f64>) {
+    ) -> (f64, f64, Vec<f64>) {
         let proportions = match sigma_proportions {
             Some(proportions) => proportions.clone(),
             None => {
@@ -60,37 +61,58 @@ impl NoiseAggregator {
             }
         };
 
-        assert!(
-            reporting_length == proportions.len(),
-            "sigma proportions array size should match the reporting length",
-        );
-
         info!(
-            "calculating percentile epsilon and sigma by len: total epsilon = {}, delta = {}, percentile_epsilon_proportion = {}, sigma_proportions = {:?}",
+            "calculating percentile epsilon, number of records epsilon and sigma by len: total epsilon = {}, delta = {}, percentile_epsilon_proportion = {}, number_of_records_proportion = {}, sigma_proportions = {:?}",
             epsilon,
             delta,
             percentile_epsilon_proportion,
+            number_of_records_proportion,
             proportions
         );
 
+        assert!(
+            reporting_length == proportions.len(),
+            "sigma proportions array size should match the reporting length",
+        );
+
+        assert!(
+            percentile_epsilon_proportion < 1.0 && percentile_epsilon_proportion > 0.0,
+            "percentile_epsilon_proportion must be > 0 and < 1"
+        );
+
+        assert!(
+            number_of_records_proportion < 1.0 && number_of_records_proportion > 0.0,
+            "number_of_records_proportion must be > 0 and < 1"
+        );
+
+        assert!(
+            number_of_records_proportion + percentile_epsilon_proportion < 1.0,
+            "(percentile_epsilon_proportion + number_of_records_proportion) must be > 0 and < 1"
+        );
+
         let t = reporting_length as f64;
         let rho = (epsilon + (2.0 / delta).ln()).sqrt() - (2.0 / delta).ln().sqrt();
         let k: f64 = proportions.iter().map(|p| 1.0 / (p * p)).sum();
         let percentile_epsilon = (2.0 * rho * percentile_epsilon_proportion / t).sqrt();
-        let base_sigma = (k / (2.0 * rho * (1.0 - percentile_epsilon_proportion))).sqrt();
+        let number_of_records_epsilon = (2.0 * rho * number_of_records_proportion).sqrt();
+        let base_sigma = (k
+            / (2.0 * rho * (1.0 - percentile_epsilon_proportion - number_of_records_proportion)))
+            .sqrt();
         let sigmas: Vec<f64> = proportions.iter().map(|p| p * base_sigma).collect();
         let lhs = ((t * percentile_epsilon * percentile_epsilon) / 2.0)
+            + ((number_of_records_epsilon * number_of_records_epsilon) / 2.0)
             + (sigmas.iter().map(|s| 1.0 / (s * s)).sum::<f64>() / 2.0);
 
         info!("percentile epsilon = {}", percentile_epsilon);
+        info!("number of records epsilon = {}", number_of_records_epsilon);
         info!("calculated sigmas = {:?}", sigmas);
 
         assert!(
             (lhs - rho).abs() <= DEFAULT_TOLERANCE,
             "something went wrong calculating DP sigmas"
         );
 
-        (percentile_epsilon, sigmas)
+        (percentile_epsilon, number_of_records_epsilon, sigmas)
     }
 
     #[inline]
@@ -427,12 +449,15 @@ impl NoiseAggregator {
         dp_parameters: &DpParameters,
         threshold: NoisyCountThreshold,
     ) -> NoiseAggregator {
-        let (percentile_epsilon, sigmas) =
-            NoiseAggregator::calc_percentile_epsilon_and_sigma_by_len(
+        let (percentile_epsilon, number_of_records_epsilon, sigmas) =
+            NoiseAggregator::calc_percentile_epsilon_number_of_records_epsilon_and_sigma_by_len(
                 reporting_length,
                 dp_parameters.epsilon,
                 dp_parameters.delta,
                 dp_parameters.percentile_epsilon_proportion,
+                dp_parameters
+                    .number_of_records_epsilon_proportion
+                    .unwrap_or(DEFAULT_NUMBER_OF_RECORDS_EPSILON_PROPORTION),
                 &dp_parameters.sigma_proportions,
             );
 
@@ -444,9 +469,7 @@ impl NoiseAggregator {
             delta: dp_parameters.delta,
             sigmas,
             threshold,
-            number_of_records_epsilon: dp_parameters
-                .number_of_records_epsilon
-                .unwrap_or(DEFAULT_NUMBER_OF_RECORDS_EPSILON),
+            number_of_records_epsilon,
         }
     }