ConfigTokenizer Class — pytorch Architecture

Source Code

c10/core/AllocatorConfig.h lines 34–129

class ConfigTokenizer {
 public:
  explicit ConfigTokenizer(const std::string& env) {
    std::string buffer;
    for (char ch : env) {
      if (ch == ',' || ch == ':' || ch == '[' || ch == ']') {
        if (!buffer.empty()) {
          config_.emplace_back(std::move(buffer));
          buffer.clear();
        }
        config_.emplace_back(1, ch);
      } else if (!std::isspace(static_cast<unsigned char>(ch))) {
        buffer += ch;
      }
    }
    if (!buffer.empty()) {
      config_.emplace_back(std::move(buffer));
    }
  }

  const std::string& operator[](size_t i) const {
    TORCH_INTERNAL_ASSERT(
        i < config_.size(), "Index out of bounds in ConfigTokenizer");
    return config_[i];
  }

  size_t size() const {
    return config_.size();
  }

  bool checkToken(size_t i, const std::string& token) const {
    checkIndex(i);
    return config_[i] == token;
  }

  size_t toSizeT(size_t i) const {
    checkIndex(i);
    return std::stoull(config_[i]);
  }

  double toDouble(size_t i) const {
    checkIndex(i);
    return std::stod(config_[i]);
  }

  bool toBool(size_t i) const {
    checkIndex(i);
    const auto& token = config_[i];
    if (token == "True") {
      return true;
    } else if (token == "False") {
      return false;
    } else {
      TORCH_CHECK_VALUE(
          false,
          "Expected 'True' or 'False' at index ",
          i,
          " in ConfigTokenizer but got '",
          token,
          "'");
    }
  }

  // Skips the current token group and returns the index of the value token.
  // Assumes the current index `i` points to a key name in a key-value pair.
  size_t skipKey(size_t i) const {
    // Expect a colon after the key
    checkToken(++i, ":");

    ++i; // Move to the value
    checkIndex(i);
    if (config_[i] != "[") {
      // Value is a single token (not a list) -> return its index
      return i;
    }

    // Skip tokens inside the list until matching ']'
    // NOLINTNEXTLINE(bugprone-inc-dec-in-conditions)
    while (++i < config_.size() && config_[i] != "]") {
    }

    TORCH_INTERNAL_ASSERT(
        i < config_.size(),
        "Expected closing bracket ']' in ConfigTokenizer but reached end of config");

    return i; // Return the index of the closing ']'
  }

 private:
  void checkIndex(size_t i) const {
    TORCH_INTERNAL_ASSERT(
        i < config_.size(), "Index out of bounds in ConfigTokenizer");
  }

  std::vector<std::string> config_;
};