tokenizer/tokenizer.hpp

Abstract tokenizer interface. More...

Namespaces

Name
sgns
sgns::neoswarm
sgns::neoswarm::core

Classes

	Name
class	sgns::neoswarm::core::Tokenizer Abstract tokenizer interface.

Detailed Description

Abstract tokenizer interface.

Date: 2026-05-06

Source code

#ifndef NEOSWARM_CORE_TOKENIZER_TOKENIZER_HPP
#define NEOSWARM_CORE_TOKENIZER_TOKENIZER_HPP

#include "common/error.hpp"
#include <memory>
#include <string>
#include <vector>

namespace sgns::neoswarm::core
{
    class Tokenizer
    {
        public:
        virtual ~Tokenizer() = default;

        virtual outcome::result<std::vector<int>> Encode( const std::string& text ) const = 0;

        virtual outcome::result<std::string> Decode( const std::vector<int>& ids ) const = 0;

        virtual bool IsEOS( int token_id ) const = 0;

        virtual int EosTokenId() const = 0;

        virtual int BosTokenId() const = 0;

        virtual size_t VocabSize() const = 0;
    };

} // namespace sgns::neoswarm::core

#endif // NEOSWARM_CORE_TOKENIZER_TOKENIZER_HPP

---

Updated on 2026-07-25 at 22:56:57 +0000