Initial checkin of Keras Optimzers and helper classes.

tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_SoftmaxCrossEntropyWithLogits.pbtxt

@@ -1,6 +1,6 @@
 op {
   graph_op_name: "SoftmaxCrossEntropyWithLogits"
   endpoint {
-    name: "nn.SoftmaxCrossEntropyWithLogits"
+    name: "nn.raw.SoftmaxCrossEntropyWithLogits"
   }
 }

tensorflow-core/tensorflow-core-api/src/bazel/api_def/api_def_SparseSoftmaxCrossEntropyWithLogits.pbtxt

@@ -1,6 +1,6 @@
 op {
   graph_op_name: "SparseSoftmaxCrossEntropyWithLogits"
   endpoint {
-    name: "nn.SparseSoftmaxCrossEntropyWithLogits"
+    name: "nn.raw.SparseSoftmaxCrossEntropyWithLogits"
   }
 }

tensorflow-core/tensorflow-core-api/src/bazel/op_generator/op_generator.cc

@@ -509,6 +509,13 @@ void GenerateOp(const OpSpec& op, const EndpointSpec& endpoint,
     RenderInterfaceImpl(op, mode, &writer);
   }
   writer.EndLine();
+
+  Variable nameVariable = Variable::Create("OP_NAME", Type::Class("String")); 
+  Javadoc name_javadoc = Javadoc::Create("The name of this op, as known by TensorFlow core engine");
+  string quoted_string = "\"" + op.graph_op_name() + "\"";
+  writer.WriteFieldWithInitializer(nameVariable, PUBLIC|STATIC|FINAL, &name_javadoc, quoted_string );
+  writer.EndLine();
+
   for (const ArgumentSpec& output : op.outputs()) {
     writer.WriteField(output.var(), PRIVATE);
   }

tensorflow-core/tensorflow-core-api/src/bazel/op_generator/source_writer.cc

@@ -238,6 +238,22 @@ SourceWriter& SourceWriter::WriteField(const Variable& field, int modifiers,
   return *this;
 }
 
+SourceWriter& SourceWriter::WriteFieldWithInitializer(const Variable& field,
+	int modifiers, const Javadoc* javadoc, const string& initializer) {
+  // If present, write field javadoc only as one brief line
+  if (javadoc != nullptr && !javadoc->brief().empty()) {
+    Append("/** ").Append(javadoc->brief()).Append(" */").EndLine();
+  }
+  WriteModifiers(modifiers);
+  if (!initializer.empty())
+    AppendType(field.type()).Append(" ").Append(field.name()).
+                            Append(" = ").Append(initializer).Append(";");
+  else
+    AppendType(field.type()).Append(" ").Append(field.name()).Append(";");
+  EndLine();
+  return *this;
+}
+
 SourceWriter& SourceWriter::WriteModifiers(int modifiers) {
   if (modifiers & PUBLIC) {
     Append("public ");

tensorflow-core/tensorflow-core-api/src/bazel/op_generator/source_writer.h

@@ -153,6 +153,9 @@ class SourceWriter {
   SourceWriter& WriteField(const Variable& field, int modifiers,
                            const Javadoc* javadoc = nullptr);
 
+  SourceWriter& WriteFieldWithInitializer(const Variable& field,
+        int modifiers, const Javadoc* javadoc = nullptr, const string& initializer = nullptr);
+
  protected:
   virtual void DoAppend(const StringPiece& str) = 0;
 
@@ -161,7 +164,7 @@ class SourceWriter {
   class TypeVisitor {
    public:
     virtual ~TypeVisitor() = default;
-    void Visit(const Type& type);
+    void Visit(const Type& type);	
 
    protected:
     virtual void DoVisit(const Type& type) = 0;

tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/NnOps.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import org.tensorflow.DataType;
 import org.tensorflow.Operand;
+import org.tensorflow.op.core.NN;
 import org.tensorflow.op.nn.AvgPool;
 import org.tensorflow.op.nn.AvgPool3d;
 import org.tensorflow.op.nn.AvgPool3dGrad;
@@ -84,11 +85,9 @@
 import org.tensorflow.op.nn.Relu6;
 import org.tensorflow.op.nn.Selu;
 import org.tensorflow.op.nn.Softmax;
-import org.tensorflow.op.nn.SoftmaxCrossEntropyWithLogits;
 import org.tensorflow.op.nn.Softsign;
 import org.tensorflow.op.nn.SpaceToBatch;
 import org.tensorflow.op.nn.SpaceToDepth;
-import org.tensorflow.op.nn.SparseSoftmaxCrossEntropyWithLogits;
 import org.tensorflow.op.nn.TopK;
 import org.tensorflow.types.TFloat32;
 import org.tensorflow.types.TInt32;
@@ -102,10 +101,13 @@
  * @see {@link Ops}
  */
 public final class NnOps {
+  public final NnRawOps raw;
+
   private final Scope scope;
 
   NnOps(Scope scope) {
     this.scope = scope;
+    raw = new NnRawOps(scope);
   }
 
   /**
@@ -1753,6 +1755,52 @@ public <T extends TNumber> Selu<T> selu(Operand<T> features) {
     return Selu.create(scope, features);
   }
 
+  /**
+   * Computes sigmoid cross entropy given `logits`.
+   *
+   *  <p>Measures the probability error in discrete classification tasks in which each class is
+   *  independent and not mutually exclusive. For instance, one could perform multilabel
+   *  classification where a picture can contain both an elephant and a dog at the same time.
+   *
+   *  <p>For brevity, let `x = logits`, `z = labels`. The logistic loss is
+   *
+   *  <pre>
+   *      z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))
+   *      = z * -log(1 / (1 + exp(-x))) + (1 - z) * -log(exp(-x) / (1 + exp(-x)))
+   *      = z * log(1 + exp(-x)) + (1 - z) * (-log(exp(-x)) + log(1 + exp(-x)))
+   *      = z * log(1 + exp(-x)) + (1 - z) * (x + log(1 + exp(-x))
+   *      = (1 - z) * x + log(1 + exp(-x))
+   *      = x - x * z + log(1 + exp(-x))
+   *  </pre>
+   *
+   *  <p>For x < 0, to avoid overflow in exp(-x), we reformulate the above
+   *
+   *  <pre>
+   *       x - x * z + log(1 + exp(-x))
+   *       = log(exp(x)) - x * z + log(1 + exp(-x))
+   *       = - x * z + log(1 + exp(x))
+   *  </pre>
+   *
+   *  <p>Hence, to ensure stability and avoid overflow, the implementation uses this equivalent
+   *  formulation
+   *
+   *  <pre>
+   *      max(x, 0) - x * z + log(1 + exp(-abs(x)))
+   *  </pre>
+   *
+   *  <p>`logits` and `labels` must have the same type and shape.
+   *
+   * @param scope The TensorFlow scope
+   * @param labels the labels
+   * @param logits the logits of type float32 or float64
+   * @param <T> the type of labels and logits
+   * @return the component-wise logistic losses.
+   */
+  public <T extends TNumber> Operand<T> sigmoidCrossEntropyWithLogits(Operand<T> labels,
+      Operand<T> logits) {
+    return NN.sigmoidCrossEntropyWithLogits(scope, labels, logits);
+  }
+
   /**
    * Computes softmax activations.
    *  <p>
@@ -1769,20 +1817,48 @@ public <T extends TNumber> Softmax<T> softmax(Operand<T> logits) {
   }
 
   /**
-   * Computes softmax cross entropy cost and gradients to backpropagate.
-   *  <p>
-   *  Inputs are the logits, not probabilities.
+   * Computes softmax cross entropy between `logits` and `labels`.
    *
-   * @param <T> data type for {@code loss()} output
-   * @param features batch_size x num_classes matrix
-   * @param labels batch_size x num_classes matrix
-   *  The caller must ensure that each batch of labels represents a valid
-   *  probability distribution.
-   * @return a new instance of SoftmaxCrossEntropyWithLogits
+   *  <p>Measures the probability error in discrete classification tasks in which the classes are
+   *  mutually exclusive (each entry is in exactly one class). For example, each CIFAR-10 image is
+   *  labeled with one and only one label: an image can be a dog or a truck, but not both.
+   *
+   *  <p>**NOTE:** While the classes are mutually exclusive, their probabilities need not be. All
+   *  that is required is that each row of `labels` is a valid probability distribution. If they are
+   *  not, the computation of the gradient will be incorrect.
+   *
+   *  <p>If using exclusive `labels` (wherein one and only one class is true at a time), see
+   *  `sparse_softmax_cross_entropy_with_logits`.
+   *
+   *  <p>Usage:
+   *
+   *  <pre>
+   *    >>> logits = [[4.0, 2.0, 1.0], [0.0, 5.0, 1.0]]
+   *    >>> labels = [[1.0, 0.0, 0.0], [0.0, 0.8, 0.2]]
+   *    >>> tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits)
+   *    <tf.Tensor: shape=(2,), dtype=float32,
+   *    numpy=array([0.16984604, 0.82474494], dtype=float32)>
+   *  </pre>
+   *
+   *  <p>Backpropagation will happen into both `logits` and `labels`. To disallow backpropagation
+   *  into `labels`, pass label tensors through `tf.stop_gradient` before feeding it to this
+   *  function.
+   *
+   * @param scope current scope
+   * @param labels Each vector along the class dimension should hold a valid probability
+   *      distribution e.g. for the case in which labels are of shape `[batch_size, num_classes]`,
+   *      each row of `labels[i]` must be a valid probability distribution.
+   * @param logits Per-label activations, typically a linear output. These activation energies are
+   *      interpreted as unnormalized log probabilities.
+   * @param axis The class dimension. -1 is the last dimension.
+   * @param <U> the data type of the logits
+   * @param <T> the number type of the operands
+   * @return the softmax cross entropy loss. Its type is the same as `logits` and its shape is the
+   *      same as `labels` except that it does not have the last dimension of `labels`.
    */
-  public <T extends TNumber> SoftmaxCrossEntropyWithLogits<T> softmaxCrossEntropyWithLogits(
-      Operand<T> features, Operand<T> labels) {
-    return SoftmaxCrossEntropyWithLogits.create(scope, features, labels);
+  public <U extends TType, T extends TNumber> Operand<T> softmaxCrossEntropyWithLogits(
+      Operand<T> labels, Operand<U> logits, int axis) {
+    return NN.softmaxCrossEntropyWithLogits(scope, labels, logits, axis);
   }
 
   /**
@@ -1974,24 +2050,22 @@ public <T extends TType> SpaceToDepth<T> spaceToDepth(Operand<T> input, Long blo
   }
 
   /**
-   * Computes softmax cross entropy cost and gradients to backpropagate.
-   *  <p>
-   *  Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
-   *  a matrix of label probabilities, but rather a single label per row
-   *  of features.  This label is considered to have probability 1.0 for the
-   *  given row.
-   *  <p>
-   *  Inputs are the logits, not probabilities.
+   * Computes sparse softmax cross entropy between `logits` and `labels`.
    *
-   * @param <T> data type for {@code loss()} output
-   * @param features batch_size x num_classes matrix
-   * @param labels batch_size vector with values in [0, num_classes).
-   *  This is the label for the given minibatch entry.
-   * @return a new instance of SparseSoftmaxCrossEntropyWithLogits
-   */
-  public <T extends TNumber, U extends TNumber> SparseSoftmaxCrossEntropyWithLogits<T> sparseSoftmaxCrossEntropyWithLogits(
-      Operand<T> features, Operand<U> labels) {
-    return SparseSoftmaxCrossEntropyWithLogits.create(scope, features, labels);
+   * @param scope current scope
+   * @param labels `Tensor` of shape `[d_0, d_1, ..., d_{r-1}]` (where `r` is rank of `labels` and
+   *      result) and dtype `int32` or `int64`. Each entry in `labels` must be an index in `[0,
+   *      num_classes)`. Other values will raise an exception when this op is run on CPU, and return
+   *      `NaN` for corresponding loss and gradient rows on GPU.
+   * @param logits Per-label activations (typically a linear output) of shape `[d_0, d_1, ...,
+   *      d_{r-1}, num_classes]` and dtype `float16`, `float32`, or `float64`. These activation
+   *      energies are interpreted as unnormalized log probabilities.
+   * @return A `Tensor` of the same shape as `labels` and of the same type as `logits` with the
+   *      softmax cross entropy loss.
+   */
+  public <T extends TNumber, U extends TNumber> Operand sparseSoftmaxCrossEntropyWithLogits(
+      Operand<T> labels, Operand<U> logits) {
+    return NN.sparseSoftmaxCrossEntropyWithLogits(scope, labels, logits);
   }
 
   /**

tensorflow-core/tensorflow-core-api/src/gen/annotations/org/tensorflow/op/NnRawOps.java

@@ -0,0 +1,74 @@
+// Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ==============================================================================
+//
+// This class has been generated, DO NOT EDIT!
+//
+package org.tensorflow.op;
+
+import org.tensorflow.Operand;
+import org.tensorflow.op.nn.raw.SoftmaxCrossEntropyWithLogits;
+import org.tensorflow.op.nn.raw.SparseSoftmaxCrossEntropyWithLogits;
+import org.tensorflow.types.family.TNumber;
+
+/**
+ * An API for building {@code nn.raw} operations as {@link Op Op}s
+ *
+ * @see {@link Ops}
+ */
+public final class NnRawOps {
+  private final Scope scope;
+
+  NnRawOps(Scope scope) {
+    this.scope = scope;
+  }
+
+  /**
+   * Computes softmax cross entropy cost and gradients to backpropagate.
+   *  <p>
+   *  Inputs are the logits, not probabilities.
+   *
+   * @param <T> data type for {@code loss()} output
+   * @param features batch_size x num_classes matrix
+   * @param labels batch_size x num_classes matrix
+   *  The caller must ensure that each batch of labels represents a valid
+   *  probability distribution.
+   * @return a new instance of SoftmaxCrossEntropyWithLogits
+   */
+  public <T extends TNumber> SoftmaxCrossEntropyWithLogits<T> softmaxCrossEntropyWithLogits(
+      Operand<T> features, Operand<T> labels) {
+    return SoftmaxCrossEntropyWithLogits.create(scope, features, labels);
+  }
+
+  /**
+   * Computes softmax cross entropy cost and gradients to backpropagate.
+   *  <p>
+   *  Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
+   *  a matrix of label probabilities, but rather a single label per row
+   *  of features.  This label is considered to have probability 1.0 for the
+   *  given row.
+   *  <p>
+   *  Inputs are the logits, not probabilities.
+   *
+   * @param <T> data type for {@code loss()} output
+   * @param features batch_size x num_classes matrix
+   * @param labels batch_size vector with values in [0, num_classes).
+   *  This is the label for the given minibatch entry.
+   * @return a new instance of SparseSoftmaxCrossEntropyWithLogits
+   */
+  public <T extends TNumber, U extends TNumber> SparseSoftmaxCrossEntropyWithLogits<T> sparseSoftmaxCrossEntropyWithLogits(
+      Operand<T> features, Operand<U> labels) {
+    return SparseSoftmaxCrossEntropyWithLogits.create(scope, features, labels);
+  }
+}

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/audio/AudioSpectrogram.java

@@ -127,6 +127,9 @@ public Output<TFloat32> asOutput() {
     return spectrogram;
   }
 
+  /** The name of this op, as known by TensorFlow core engine */
+  public static final String OP_NAME = "AudioSpectrogram";
+
   private Output<TFloat32> spectrogram;
 
   private AudioSpectrogram(Operation operation) {

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/audio/DecodeWav.java

@@ -132,6 +132,9 @@ public Output<TInt32> sampleRate() {
     return sampleRate;
   }
 
+  /** The name of this op, as known by TensorFlow core engine */
+  public static final String OP_NAME = "DecodeWav";
+
   private Output<TFloat32> audio;
   private Output<TInt32> sampleRate;
 

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/audio/EncodeWav.java

@@ -72,6 +72,9 @@ public Output<TString> asOutput() {
     return contents;
   }
 
+  /** The name of this op, as known by TensorFlow core engine */
+  public static final String OP_NAME = "EncodeWav";
+
   private Output<TString> contents;
 
   private EncodeWav(Operation operation) {

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/audio/Mfcc.java

@@ -165,6 +165,9 @@ public Output<TFloat32> asOutput() {
     return output;
   }
 
+  /** The name of this op, as known by TensorFlow core engine */
+  public static final String OP_NAME = "Mfcc";
+
   private Output<TFloat32> output;
 
   private Mfcc(Operation operation) {

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/bitwise/BitwiseAnd.java

@@ -84,6 +84,9 @@ public Output<T> asOutput() {
     return z;
   }
 
+  /** The name of this op, as known by TensorFlow core engine */
+  public static final String OP_NAME = "BitwiseAnd";
+
   private Output<T> z;
 
   private BitwiseAnd(Operation operation) {

tensorflow-core/tensorflow-core-api/src/gen/java/org/tensorflow/op/bitwise/BitwiseOr.java

@@ -84,6 +84,9 @@ public Output<T> asOutput() {
     return z;
   }
 
+  /** The name of this op, as known by TensorFlow core engine */
+  public static final String OP_NAME = "BitwiseOr";
+
   private Output<T> z;
 
   private BitwiseOr(Operation operation) {