Add a tokenizer (#1641)

Summary:
Pull Request resolved: #1641

Add a tokenizer in examples. This needs to consume the artifact generated by `tokenizer.py`.

Reviewed By: mikekgfb

Differential Revision: D52894344

fbshipit-source-id: e3f03fa5cd2d2607ef2bfd371125de1f72adc968

Author: larryliu0820

examples/models/llama2/tokenizer/targets.bzl

@@ -0,0 +1,33 @@
+load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
+
+def define_common_targets():
+    runtime.cxx_library(
+        name = "tokenizer_lib",
+        srcs = ["tokenizer.cpp"],
+        headers = ["tokenizer.h"],
+        exported_deps = [
+            "//executorch/runtime/core/exec_aten:lib",
+            "//executorch/runtime/kernel:kernel_includes",
+        ],
+        visibility = [
+            "//executorch/...",
+        ],
+    )
+
+    if not runtime.is_oss:
+        # no resources support
+        runtime.export_file(
+            name = "tokenizer_file",
+            src = "test/test.bin",
+        )
+
+        runtime.cxx_test(
+            name = "test_tokenizer_cpp",
+            srcs = ["test/test_tokenizer.cpp"],
+            deps = [
+                ":tokenizer_lib",
+                "//executorch/codegen:macros",
+                "fbsource//xplat/tools/cxx:resources",
+            ],
+            resources = [":tokenizer_file"],
+        )

examples/models/llama2/tokenizer/test/test.bin

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <executorch/examples/models/llama2/tokenizer/tokenizer.h>
+#include <executorch/runtime/platform/runtime.h>
+#include <gtest/gtest.h>
+#include "tools/cxx/Resources.h"
+
+using namespace ::testing;
+
+namespace torch {
+namespace executor {
+
+class TokenizerExtensionTest : public ::testing::Test {
+ public:
+  void SetUp() override {
+    torch::executor::runtime_init();
+    modelPath_ =
+        build::getResourcePath(
+            "executorch/examples/models/llama2/tokenizer/test/test.bin")
+            .string();
+    tokenizer_ = std::make_unique<Tokenizer>(32000);
+  }
+
+  std::unique_ptr<Tokenizer> tokenizer_;
+  std::string modelPath_;
+};
+
+TEST_F(TokenizerExtensionTest, EncodeWithoutLoadFails) {
+  Error error = tokenizer_->encode("hello world", 0, 0, nullptr, nullptr);
+  EXPECT_EQ(error, Error::NotSupported);
+}
+
+TEST_F(TokenizerExtensionTest, DecodeWithoutLoadFails) {
+  auto result = tokenizer_->decode(0, 0);
+  EXPECT_EQ(result.error(), Error::NotSupported);
+}
+
+TEST_F(TokenizerExtensionTest, TokenizerVocabSizeIsExpected) {
+  Error res = tokenizer_->load(modelPath_.c_str());
+  EXPECT_EQ(res, Error::Ok);
+  // test.bin has vocab size 0 but the tokenizer respects the vocab size being
+  // passed in and add placeholder tokens.
+  EXPECT_EQ(tokenizer_->vocab_size(), 32000);
+  EXPECT_EQ(tokenizer_->bos_tok(), 1);
+  EXPECT_EQ(tokenizer_->eos_tok(), 2);
+}
+
+} // namespace executor
+} // namespace torch