Add Regularizers 1 (#216)

Author: JimClarke5

tensorflow-framework/src/main/java/org/tensorflow/framework/regularizers/L1.java

@@ -0,0 +1,47 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.regularizers;
+
+import org.tensorflow.op.Ops;
+
+/**
+ * A regularizer that applies an L1 or Lasso(least absolute shrinkage and selection operator)
+ * Regression, regularization penalty.
+ *
+ * <p>The L1 regularization penalty is computed as: <code>loss = l1 * reduceSum(abs(x))</code>
+ */
+public class L1 extends L1L2 {
+
+  /**
+   * Create a regularizer that applies an L1 regularization penalty of {@link
+   * #DEFAULT_REGULARIZATION_PENALTY}
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public L1(Ops tf) {
+    this(tf, DEFAULT_REGULARIZATION_PENALTY);
+  }
+
+  /**
+   * Create a regularizer that applies an L1 regularization penalty
+   *
+   * @param tf the TensorFlow Ops
+   * @param l1 the L1 regularization penalty
+   * @throws IllegalArgumentException if the l1 regularization factor is NaN or is infinite.
+   */
+  public L1(Ops tf, float l1) {
+    super(tf, l1, 0f);
+  }
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/regularizers/L1L2.java

@@ -0,0 +1,120 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.regularizers;
+
+import org.tensorflow.Operand;
+import org.tensorflow.framework.losses.impl.LossesHelper;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * A regularizer that applies both L1 and L2 regularization penalties.
+ *
+ * <p>The L1 regularization penalty is computed as:
+ *
+ * <pre>loss = l1 * reduceSum(abs(x))</pre>
+ *
+ * <p>The L2 regularization penalty is computed as
+ *
+ * <pre>loss = l2 * reduceSum(square(x))</pre>
+ *
+ */
+public class L1L2 extends Regularizer {
+
+  private final float l1;
+  private final float l2;
+
+  /**
+   * Creates an L1L2 regularizer with no l1 or l2 penalty with zero penalty
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public L1L2(Ops tf) {
+    this(tf, DEFAULT_REGULARIZATION_PENALTY, DEFAULT_REGULARIZATION_PENALTY);
+  }
+
+  /**
+   * Creates an L1L2 regularizer
+   *
+   * @param tf the TensorFlow Ops
+   * @param l1 L1 regularization factor, if null it is set to 0.
+   * @param l2 L2 regularization factor, if null it is set to 0.
+   * @throws IllegalArgumentException if the l1 or l2 regularization factor is {@link Float#isNaN}
+   *     of {@link Float#isInfinite}
+   */
+  public L1L2(Ops tf, float l1, float l2) {
+    super(tf);
+    if (Float.isNaN(l1) || Float.isInfinite(l1)) {
+      throw new IllegalArgumentException(
+          String.format(
+              "L1 Value: %f is not a valid regularization penalty number, a positive/negative infinity or NaN is not a property value",
+              l1));
+    }
+    this.l1 = l1;
+
+    if (Float.isNaN(l2) || Float.isInfinite(l2)) {
+      throw new IllegalArgumentException(
+          String.format(
+              "L2 Value: %f is not a valid regularization penalty number, a positive/negative infinity or NaN is not a property value",
+              l2));
+    }
+    this.l2 = l2;
+  }
+
+
+  /** {@inheritDoc} */
+  @Override
+  public <R extends TNumber> Operand<R> call(Operand<R> input) {
+    Ops tf = getTF();
+    if (this.getL1() == 0f && this.getL2() == 0f) {
+      return tf.dtypes.cast(tf.constant(0), input.type());
+    }
+    Operand<R> regularization = tf.dtypes.cast(tf.constant(0), input.type());
+
+    if (this.getL1() != 0.f) {
+      Operand<R> l1Op = tf.dtypes.cast(tf.constant(this.getL1()), input.type());
+      Operand<R> abs = tf.math.abs(input);
+      Operand<R> reduceSum = tf.reduceSum(abs, LossesHelper.allAxes(tf, input));
+      regularization = tf.math.add(regularization, tf.math.mul(l1Op, reduceSum));
+    }
+
+    if (this.getL2() != 0.f) {
+      Operand<R> l2Op = tf.dtypes.cast(tf.constant(this.getL2()), input.type());
+      Operand<R> sqr = tf.math.square(input);
+      Operand<R> reduceSum = tf.reduceSum(sqr, LossesHelper.allAxes(tf, input));
+      regularization = tf.math.add(regularization, tf.math.mul(l2Op, reduceSum));
+    }
+
+    return regularization;
+  }
+
+  /**
+   * Gets the L1 regularization factor
+   *
+   * @return the L1 regularization factor
+   */
+  public float getL1() {
+    return l1;
+  }
+
+  /**
+   * Gets the L2 regularization factor
+   *
+   * @return the L2 regularization factor
+   */
+  public float getL2() {
+    return l2;
+  }
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/regularizers/L2.java

@@ -0,0 +1,46 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.regularizers;
+
+import org.tensorflow.op.Ops;
+
+/**
+ * A regularizer that applies a L2 (Ridge Regression) regularization penalty.
+ *
+ * <p>The L2 regularization penalty is computed as: <code>loss = l2 * reduceSum(square(x))</code>
+ */
+public class L2 extends L1L2 {
+
+  /**
+   * Create a regularizer that applies an L2 regularization penalty of {@link
+   * #DEFAULT_REGULARIZATION_PENALTY}
+   *
+   * @param tf the TensorFlow Ops
+   */
+  public L2(Ops tf) {
+    this(tf, DEFAULT_REGULARIZATION_PENALTY);
+  }
+
+  /**
+   * Create a regularizer that applies an L1 regularization penalty
+   *
+   * @param tf the TensorFlow Ops
+   * @param l2 the L2 regularization penalty
+   * @throws IllegalArgumentException if the l2 regularization factor is NaN or is infinite.
+   */
+  public L2(Ops tf, float l2) {
+    super(tf, 0f, l2);
+  }
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/regularizers/Regularizer.java

@@ -0,0 +1,91 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.regularizers;
+
+import org.tensorflow.Operand;
+import org.tensorflow.framework.losses.Loss;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * Base class for Regularizers
+ *
+ * <p>Regularizers allow you to apply penalties on layer parameters or layer activity during
+ * optimization. These penalties are summed into the loss function that the network optimizes.
+ */
+public abstract class Regularizer {
+
+  public static final float DEFAULT_REGULARIZATION_PENALTY = 0.01f;
+
+  private final Ops tf;
+  private final String name;
+
+  /**
+   * Creates a Regularizer, using {@link Class#getSimpleName()} for the name
+   *
+   * @param tf the TensorFlow ops.
+   */
+  protected Regularizer(Ops tf) {
+    this(tf, null);
+  }
+  /**
+   * Creates a Regularizer
+   *
+   * @param tf the TensorFlow ops.
+   * @param name the name of this regularizer, if null use {@link Class#getSimpleName()} for the
+   *     name.
+   */
+  protected Regularizer(Ops tf, String name) {
+    this.tf = tf;
+    this.name = name == null ? this.getClass().getSimpleName() : name;
+  }
+
+  /**
+   * Returns this Regularizer as a Loss This is a convenience to use regularize a loss. Only
+   * sampleWeights are applied to the regularizer.
+   *
+   * @return this Regularizer as a Loss
+   */
+  public Loss asLoss() {
+    return new RegularizerLoss(this.tf, this);
+  }
+
+  /**
+   * Computes a regularization penalty from an input.
+   *
+   * @param input the weighted input
+   * @return the result of computing the regularization penalty
+   * @param <R> the data type of the input and result
+   */
+  public abstract <R extends TNumber> Operand<R> call(Operand<R> input);
+
+  /**
+   * Gets the TensorFlow Ops
+   *
+   * @return the TensorFlow Ops
+   */
+  public Ops getTF() {
+    return tf;
+  }
+
+  /**
+   * Gets the name for this regularizer
+   *
+   * @return the name for this regularizer
+   */
+  public String getName() {
+    return name;
+  }
+}

tensorflow-framework/src/main/java/org/tensorflow/framework/regularizers/RegularizerLoss.java

@@ -0,0 +1,64 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=======================================================================*/
+package org.tensorflow.framework.regularizers;
+
+import org.tensorflow.Operand;
+import org.tensorflow.framework.losses.Loss;
+import org.tensorflow.op.Ops;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * A Regularizer call wrapped as a Loss instance
+ *
+ * <p>This class facilitates using a regularizer as a loss, only <code>sampleWeights</code> are
+ * regularized.
+ */
+class RegularizerLoss extends Loss {
+
+  private final Regularizer regularizer;
+
+  /**
+   * Creates a Loss using {@link Class#getSimpleName()} as the name and a Loss Reduction of {@link
+   * Loss#REDUCTION_DEFAULT}
+   *
+   * @param tf the TensorFlow Ops
+   * @param regularizer the regularizer used to calculate the loss
+   */
+  public RegularizerLoss(Ops tf, Regularizer regularizer) {
+    this(tf, null, regularizer);
+  }
+
+  /**
+   * Creates a Loss using a Loss Reduction of {@link Loss#REDUCTION_DEFAULT}
+   *
+   * @param tf the TensorFlow Ops
+   * @param name the name of this Loss, if null the name will be {@link Class#getSimpleName()}.
+   * @param regularizer the regularizer used to calculate the loss
+   */
+  public RegularizerLoss(Ops tf, String name, Regularizer regularizer) {
+    super(tf, name);
+    this.regularizer = regularizer;
+  }
+
+  /** {@inheritDoc} */
+  @Override
+  public <T extends TNumber> Operand<T> call(
+      Operand<? extends TNumber> labels, Operand<T> predictions, Operand<T> sampleWeights) {
+    if (sampleWeights == null) {
+      throw new IllegalArgumentException("sampleWeights cannot be null");
+    }
+    return regularizer.call(sampleWeights);
+  }
+}