#ifndef LANG_TOKENIZER_COMMON_H
#define LANG_TOKENIZER_COMMON_H
void
lang_tokenizer_delete_trailing_partial_char(const unsigned char *data,
                                           size_t *len);
void
lang_tokenizer_delete_trailing_invalid_char(const unsigned char *data,
		   size_t *len);
#endif