//

// Copyright Aliaksei Levin (levlam@telegram.org), Arseny Smirnov (arseny30@gmail.com) 2014-2026

//

// Distributed under the Boost Software License, Version 1.0. (See accompanying

// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

//

#include "td/telegram/MessageEntity.h"

#include "td/telegram/Dependencies.h"

#include "td/telegram/DialogManager.h"

#include "td/telegram/LinkManager.h"

#include "td/telegram/misc.h"

#include "td/telegram/OptionManager.h"

#include "td/telegram/SecretChatLayer.h"

#include "td/telegram/StickersManager.h"

#include "td/telegram/Td.h"

#include "td/telegram/UserManager.h"

#include "td/actor/MultiPromise.h"

#include "td/utils/algorithm.h"

#include "td/utils/HashTableUtils.h"

#include "td/utils/logging.h"

#include "td/utils/misc.h"

#include "td/utils/Promise.h"

#include "td/utils/SliceBuilder.h"

#include "td/utils/StringBuilder.h"

#include "td/utils/unicode.h"

#include "td/utils/utf8.h"

#include <algorithm>

#include <cstring>

#include <limits>

#include <tuple>

namespace td {

static Result<int32> get_date_flags(const string &date_format) {

if (date_format == "r" || date_format == "R") {

return MessageEntity::DateFlags::Relative;

}

int32 date_flags = 0;

for (auto c : date_format) {

switch (c) {

case 't':

date_flags |= MessageEntity::DateFlags::ShortTime;

break;

case 'T':

date_flags |= MessageEntity::DateFlags::LongTime;

break;

case 'd':

date_flags |= MessageEntity::DateFlags::ShortDate;

break;

case 'D':

date_flags |= MessageEntity::DateFlags::LongDate;

break;

case 'w':

case 'W':

date_flags |= MessageEntity::DateFlags::DayOfWeek;

break;

default:

return Status::Error(400, "Invalid date format used");

}

}

return std::move(date_flags);

}

int MessageEntity::get_type_priority(Type type) {

static const int priorities[] = {50 /*Mention*/,

50 /*Hashtag*/,

50 /*BotCommand*/,

50 /*Url*/,

50 /*EmailAddress*/,

90 /*Bold*/,

91 /*Italic*/,

20 /*Code*/,

11 /*Pre*/,

10 /*PreCode*/,

49 /*TextUrl*/,

49 /*MentionName*/,

50 /*Cashtag*/,

50 /*PhoneNumber*/,

92 /*Underline*/,

93 /*Strikethrough*/,

0 /*BlockQuote*/,

50 /*BankCardNumber*/,

50 /*MediaTimestamp*/,

94 /*Spoiler*/,

99 /*CustomEmoji*/,

0 /*ExpandableBlockQuote*/,

30 /*FormattedDate*/};

static_assert(sizeof(priorities) / sizeof(priorities[0]) == static_cast<size_t>(MessageEntity::Type::Size), "");

return priorities[static_cast<int32>(type)];

}

StringBuilder &operator<<(StringBuilder &string_builder, const MessageEntity::Type &message_entity_type) {

switch (message_entity_type) {

case MessageEntity::Type::Mention:

return string_builder << "Mention";

case MessageEntity::Type::Hashtag:

return string_builder << "Hashtag";

case MessageEntity::Type::BotCommand:

return string_builder << "BotCommand";

case MessageEntity::Type::Url:

return string_builder << "Url";

case MessageEntity::Type::EmailAddress:

return string_builder << "EmailAddress";

case MessageEntity::Type::Bold:

return string_builder << "Bold";

case MessageEntity::Type::Italic:

return string_builder << "Italic";

case MessageEntity::Type::Underline:

return string_builder << "Underline";

case MessageEntity::Type::Strikethrough:

return string_builder << "Strikethrough";

case MessageEntity::Type::BlockQuote:

return string_builder << "BlockQuote";

case MessageEntity::Type::Code:

return string_builder << "Code";

case MessageEntity::Type::Pre:

return string_builder << "Pre";

case MessageEntity::Type::PreCode:

return string_builder << "PreCode";

case MessageEntity::Type::TextUrl:

return string_builder << "TextUrl";

case MessageEntity::Type::MentionName:

return string_builder << "MentionName";

case MessageEntity::Type::Cashtag:

return string_builder << "Cashtag";

case MessageEntity::Type::PhoneNumber:

return string_builder << "PhoneNumber";

case MessageEntity::Type::BankCardNumber:

return string_builder << "BankCardNumber";

case MessageEntity::Type::MediaTimestamp:

return string_builder << "MediaTimestamp";

case MessageEntity::Type::Spoiler:

return string_builder << "Spoiler";

case MessageEntity::Type::CustomEmoji:

return string_builder << "CustomEmoji";

case MessageEntity::Type::ExpandableBlockQuote:

return string_builder << "ExpandableBlockQuote";

case MessageEntity::Type::FormattedDate:

return string_builder << "Date";

default:

UNREACHABLE();

return string_builder << "Impossible";

}

}

StringBuilder &operator<<(StringBuilder &string_builder, const MessageEntity &message_entity) {

string_builder << '[' << message_entity.type << ", offset = " << message_entity.offset

<< ", length = " << message_entity.length;

if (message_entity.media_timestamp >= 0) {

string_builder << ", media_timestamp = \"" << message_entity.media_timestamp << "\"";

}

if (!message_entity.argument.empty()) {

string_builder << ", argument = \"" << message_entity.argument << "\"";

}

if (message_entity.user_id.is_valid()) {

string_builder << ", " << message_entity.user_id;

}

if (message_entity.custom_emoji_id.is_valid()) {

string_builder << ", " << message_entity.custom_emoji_id;

}

if (message_entity.date != 0) {

string_builder << ", at " << message_entity.date << " with flags " << message_entity.date_flags;

}

string_builder << ']';

return string_builder;

}

tl_object_ptr<td_api::TextEntityType> MessageEntity::get_text_entity_type_object(

const UserManager *user_manager) const {

switch (type) {

case MessageEntity::Type::Mention:

return make_tl_object<td_api::textEntityTypeMention>();

case MessageEntity::Type::Hashtag:

return make_tl_object<td_api::textEntityTypeHashtag>();

case MessageEntity::Type::BotCommand:

return make_tl_object<td_api::textEntityTypeBotCommand>();

case MessageEntity::Type::Url:

return make_tl_object<td_api::textEntityTypeUrl>();

case MessageEntity::Type::EmailAddress:

return make_tl_object<td_api::textEntityTypeEmailAddress>();

case MessageEntity::Type::Bold:

return make_tl_object<td_api::textEntityTypeBold>();

case MessageEntity::Type::Italic:

return make_tl_object<td_api::textEntityTypeItalic>();

case MessageEntity::Type::Underline:

return make_tl_object<td_api::textEntityTypeUnderline>();

case MessageEntity::Type::Strikethrough:

return make_tl_object<td_api::textEntityTypeStrikethrough>();

case MessageEntity::Type::BlockQuote:

return make_tl_object<td_api::textEntityTypeBlockQuote>();

case MessageEntity::Type::Code:

return make_tl_object<td_api::textEntityTypeCode>();

case MessageEntity::Type::Pre:

return make_tl_object<td_api::textEntityTypePre>();

case MessageEntity::Type::PreCode:

return make_tl_object<td_api::textEntityTypePreCode>(argument);

case MessageEntity::Type::TextUrl:

return make_tl_object<td_api::textEntityTypeTextUrl>(argument);

case MessageEntity::Type::MentionName:

return make_tl_object<td_api::textEntityTypeMentionName>(

user_manager == nullptr ? user_id.get()

: user_manager->get_user_id_object(user_id, "textEntityTypeMentionName"));

case MessageEntity::Type::Cashtag:

return make_tl_object<td_api::textEntityTypeCashtag>();

case MessageEntity::Type::PhoneNumber:

return make_tl_object<td_api::textEntityTypePhoneNumber>();

case MessageEntity::Type::BankCardNumber:

return make_tl_object<td_api::textEntityTypeBankCardNumber>();

case MessageEntity::Type::MediaTimestamp:

return make_tl_object<td_api::textEntityTypeMediaTimestamp>(media_timestamp);

case MessageEntity::Type::Spoiler:

return make_tl_object<td_api::textEntityTypeSpoiler>();

case MessageEntity::Type::CustomEmoji:

return make_tl_object<td_api::textEntityTypeCustomEmoji>(custom_emoji_id.get());

case MessageEntity::Type::ExpandableBlockQuote:

return make_tl_object<td_api::textEntityTypeExpandableBlockQuote>();

case MessageEntity::Type::FormattedDate: {

td_api::object_ptr<td_api::DateTimeFormattingType> formatting_type;

if ((date_flags & DateFlags::Relative) != 0) {

formatting_type = td_api::make_object<td_api::dateTimeFormattingTypeRelative>();

} else if (date_flags != 0) {

td_api::object_ptr<td_api::DateTimePartPrecision> time_precision;

if ((date_flags & DateFlags::ShortTime) != 0) {

time_precision = td_api::make_object<td_api::dateTimePartPrecisionShort>();

} else if ((date_flags & DateFlags::LongTime) != 0) {

time_precision = td_api::make_object<td_api::dateTimePartPrecisionLong>();

} else {

time_precision = td_api::make_object<td_api::dateTimePartPrecisionNone>();

}

td_api::object_ptr<td_api::DateTimePartPrecision> date_precision;

if ((date_flags & DateFlags::ShortDate) != 0) {

date_precision = td_api::make_object<td_api::dateTimePartPrecisionShort>();

} else if ((date_flags & DateFlags::LongDate) != 0) {

date_precision = td_api::make_object<td_api::dateTimePartPrecisionLong>();

} else {

date_precision = td_api::make_object<td_api::dateTimePartPrecisionNone>();

}

auto show_day_of_week = (date_flags & DateFlags::DayOfWeek) != 0;

formatting_type = td_api::make_object<td_api::dateTimeFormattingTypeAbsolute>(

std::move(time_precision), std::move(date_precision), show_day_of_week);

}

return make_tl_object<td_api::textEntityTypeDateTime>(date, std::move(formatting_type));

}

default:

UNREACHABLE();

return nullptr;

}

}

tl_object_ptr<td_api::textEntity> MessageEntity::get_text_entity_object(const UserManager *user_manager) const {

return make_tl_object<td_api::textEntity>(offset, length, get_text_entity_type_object(user_manager));

}

vector<tl_object_ptr<td_api::textEntity>> get_text_entities_object(const UserManager *user_manager,

const vector<MessageEntity> &entities,

bool skip_bot_commands, int32 max_media_timestamp) {

vector<tl_object_ptr<td_api::textEntity>> result;

result.reserve(entities.size());

for (auto &entity : entities) {

if (skip_bot_commands && entity.type == MessageEntity::Type::BotCommand) {

continue;

}

if (entity.type == MessageEntity::Type::MediaTimestamp && max_media_timestamp < entity.media_timestamp) {

continue;

}

auto entity_object = entity.get_text_entity_object(user_manager);

if (entity_object->type_ != nullptr) {

result.push_back(std::move(entity_object));

}

}

return result;

}

StringBuilder &operator<<(StringBuilder &string_builder, const FormattedText &text) {

return string_builder << '"' << text.text << "\" with entities " << text.entities;

}

td_api::object_ptr<td_api::formattedText> get_formatted_text_object(const UserManager *user_manager,

const FormattedText &text, bool skip_bot_commands,

int32 max_media_timestamp) {

return td_api::make_object<td_api::formattedText>(

text.text, get_text_entities_object(user_manager, text.entities, skip_bot_commands, max_media_timestamp));

}

static bool is_word_character(uint32 code) {

switch (get_unicode_simple_category(code)) {

case UnicodeSimpleCategory::Letter:

case UnicodeSimpleCategory::DecimalNumber:

case UnicodeSimpleCategory::Number:

return true;

default:

return code == '_';

}

}

/*

static bool is_word_boundary(uint32 a, uint32 b) {

return is_word_character(a) ^ is_word_character(b);

}

*/

static bool is_alpha_digit(uint32 code) {

return ('0' <= code && code <= '9') || ('a' <= code && code <= 'z') || ('A' <= code && code <= 'Z');

}

static bool is_alpha_digit_or_underscore(uint32 code) {

return is_alpha_digit(code) || code == '_';

}

static bool is_alpha_digit_or_underscore_or_minus(uint32 code) {

return is_alpha_digit_or_underscore(code) || code == '-';

}

// This functions just implements corresponding regexps

// All other fixes will be in other functions

static vector<Slice> match_mentions(Slice str) {

vector<Slice> result;

const unsigned char *begin = str.ubegin();

const unsigned char *end = str.uend();

const unsigned char *ptr = begin;

// '/(?<=\B)@([a-zA-Z0-9_]{2,32})(?=\b)/u'

while (true) {

ptr = static_cast<const unsigned char *>(std::memchr(ptr, '@', narrow_cast<int32>(end - ptr)));

if (ptr == nullptr) {

break;

}

if (ptr != begin) {

uint32 prev;

next_utf8_unsafe(prev_utf8_unsafe(ptr), &prev);

if (is_word_character(prev)) {

ptr++;

continue;

}

}

auto mention_begin = ++ptr;

while (ptr != end && is_alpha_digit_or_underscore(*ptr)) {

ptr++;

}

auto mention_end = ptr;

auto mention_size = mention_end - mention_begin;

if (mention_size < 2 || mention_size > 32) {

continue;

}

uint32 next = 0;

if (ptr != end) {

next_utf8_unsafe(ptr, &next);

}

if (is_word_character(next)) {

continue;

}

result.emplace_back(mention_begin - 1, mention_end);

}

return result;

}

static vector<Slice> match_bot_commands(Slice str) {

vector<Slice> result;

const unsigned char *begin = str.ubegin();

const unsigned char *end = str.uend();

const unsigned char *ptr = begin;

// '/(?<!\b|[\/<>])\/([a-zA-Z0-9_]{1,64})(?:@([a-zA-Z0-9_]{3,32}))?(?!\B|[\/<>])/u'

while (true) {

ptr = static_cast<const unsigned char *>(std::memchr(ptr, '/', narrow_cast<int32>(end - ptr)));

if (ptr == nullptr) {

break;

}

if (ptr != begin) {

uint32 prev;

next_utf8_unsafe(prev_utf8_unsafe(ptr), &prev);

if (is_word_character(prev) || prev == '/' || prev == '<' || prev == '>') {

ptr++;

continue;

}

}

auto command_begin = ++ptr;

while (ptr != end && is_alpha_digit_or_underscore(*ptr)) {

ptr++;

}

auto command_end = ptr;

auto command_size = command_end - command_begin;

if (command_size < 1 || command_size > 64) {

continue;

}

if (ptr != end && *ptr == '@') {

auto mention_begin = ++ptr;

while (ptr != end && is_alpha_digit_or_underscore(*ptr)) {

ptr++;

}

auto mention_end = ptr;

auto mention_size = mention_end - mention_begin;

if (mention_size < 3 || mention_size > 32) {

continue;

}

command_end = ptr;

}

uint32 next = 0;

if (ptr != end) {

next_utf8_unsafe(ptr, &next);

}

if (is_word_character(next) || next == '/' || next == '<' || next == '>') {

continue;

}

result.emplace_back(command_begin - 1, command_end);

}

return result;

}

static bool is_hashtag_letter(uint32 c, UnicodeSimpleCategory &category) {

category = get_unicode_simple_category(c);

if (c == '_' || c == 0x200c || c == 0xb7 || (0xd80 <= c && c <= 0xdff)) {

return true;

}

switch (category) {

case UnicodeSimpleCategory::DecimalNumber:

case UnicodeSimpleCategory::Letter:

return true;

default:

return false;

}

}

static vector<Slice> match_hashtags(Slice str) {

vector<Slice> result;

const unsigned char *begin = str.ubegin();

const unsigned char *end = str.uend();

const unsigned char *ptr = begin;

// '/(?<=^|[^is_hashtag_letter])#([is_hashtag_letter]{1,256})(?:@([a-zA-Z0-9_]{3,32}))?(?![is_hashtag_letter]*#)/u'

// and at least one letter

UnicodeSimpleCategory category;

while (true) {

ptr = static_cast<const unsigned char *>(std::memchr(ptr, '#', narrow_cast<int32>(end - ptr)));

if (ptr == nullptr) {

break;

}

if (ptr != begin) {

uint32 prev;

next_utf8_unsafe(prev_utf8_unsafe(ptr), &prev);

if (is_hashtag_letter(prev, category)) {

ptr++;

continue;

}

}

auto hashtag_begin = ++ptr;

size_t hashtag_size = 0;

const unsigned char *hashtag_end = nullptr;

bool was_letter = false;

while (ptr != end) {

uint32 code;

auto next_ptr = next_utf8_unsafe(ptr, &code);

if (!is_hashtag_letter(code, category)) {

break;

}

ptr = next_ptr;

if (hashtag_size == 255) {

hashtag_end = ptr;

}

if (hashtag_size != 256) {

was_letter |= category == UnicodeSimpleCategory::Letter;

hashtag_size++;

}

}

if (!hashtag_end) {

hashtag_end = ptr;

}

if (hashtag_size < 1) {

continue;

}

if (hashtag_end == ptr && ptr != end && ptr[0] == '@') {

auto username_end = ptr + 1;

while (username_end != end && username_end - ptr < 33 && is_alpha_digit_or_underscore(*username_end)) {

username_end++;

}

auto length = username_end - ptr - 1;

if (length >= 3) {

ptr = username_end;

hashtag_end = username_end;

}

}

if (ptr != end && ptr[0] == '#') {

continue;

}

if (!was_letter) {

continue;

}

result.emplace_back(hashtag_begin - 1, hashtag_end);

}

return result;

}

static vector<Slice> match_cashtags(Slice str) {

vector<Slice> result;

const unsigned char *begin = str.ubegin();

const unsigned char *end = str.uend();

const unsigned char *ptr = begin;

// '/(?<=^|[^$is_hashtag_letter])\$(1INCH|[A-Z]{1,8})(?:@([a-zA-Z0-9_]{3,32}))?(?![$is_hashtag_letter])/u'

UnicodeSimpleCategory category;

while (true) {

ptr = static_cast<const unsigned char *>(std::memchr(ptr, '$', narrow_cast<int32>(end - ptr)));

if (ptr == nullptr) {

break;

}

if (ptr != begin) {

uint32 prev;

next_utf8_unsafe(prev_utf8_unsafe(ptr), &prev);

if (is_hashtag_letter(prev, category) || prev == '$') {

ptr++;

continue;

}

}

auto cashtag_begin = ++ptr;

if (end - ptr >= 5 && Slice(ptr, ptr + 5) == Slice("1INCH")) {

ptr += 5;

} else {

while (ptr != end && 'Z' >= *ptr && *ptr >= 'A') {

ptr++;

}

}

auto cashtag_end = ptr;

auto cashtag_size = cashtag_end - cashtag_begin;

if (cashtag_size < 1 || cashtag_size > 8) {

continue;

}

if (ptr != end && ptr[0] == '@') {

auto username_end = ptr + 1;

while (username_end != end && is_alpha_digit_or_underscore(*username_end)) {

username_end++;

}

auto length = username_end - ptr - 1;

if (length >= 3 && length <= 32) {

cashtag_end = username_end;

ptr = username_end;

}

}

if (cashtag_end != end) {

uint32 code;

next_utf8_unsafe(ptr, &code);

if (is_hashtag_letter(code, category) || code == '$') {

continue;

}

}

result.emplace_back(cashtag_begin - 1, cashtag_end);

}

return result;

}

static vector<Slice> match_media_timestamps(Slice str) {

vector<Slice> result;

const unsigned char *begin = str.ubegin();

const unsigned char *end = str.uend();

const unsigned char *ptr = begin;

while (true) {

ptr = static_cast<const unsigned char *>(std::memchr(ptr, ':', narrow_cast<int32>(end - ptr)));

if (ptr == nullptr) {

break;

}

auto media_timestamp_begin = ptr;

while (media_timestamp_begin != begin &&

(media_timestamp_begin[-1] == ':' || is_digit(media_timestamp_begin[-1]))) {

media_timestamp_begin--;

}

auto media_timestamp_end = ptr;

while (media_timestamp_end + 1 != end && (media_timestamp_end[1] == ':' || is_digit(media_timestamp_end[1]))) {

media_timestamp_end++;

}

media_timestamp_end++;

if (media_timestamp_begin != ptr && media_timestamp_end != ptr + 1 && is_digit(ptr[1])) {

ptr = media_timestamp_end;

if (media_timestamp_begin != begin) {

uint32 prev;

next_utf8_unsafe(prev_utf8_unsafe(media_timestamp_begin), &prev);

if (is_word_character(prev)) {

continue;

}

}

if (media_timestamp_end != end) {

uint32 next;

next_utf8_unsafe(media_timestamp_end, &next);

if (is_word_character(next)) {

continue;

}

}

result.emplace_back(media_timestamp_begin, media_timestamp_end);

} else {

ptr = media_timestamp_end;

}

}

return result;

}

static vector<Slice> match_bank_card_numbers(Slice str) {

vector<Slice> result;

const unsigned char *begin = str.ubegin();

const unsigned char *end = str.uend();

const unsigned char *ptr = begin;

// '/(?<=^|[^+_\pL\d-.,])[\d -]{13,}([^_\pL\d-]|$)/'

while (true) {

while (ptr != end && !is_digit(*ptr)) {

ptr++;

}

if (ptr == end) {

break;

}

if (ptr != begin) {

uint32 prev;

next_utf8_unsafe(prev_utf8_unsafe(ptr), &prev);

if (prev == '.' || prev == ',' || prev == '+' || prev == '-' || prev == '_' ||

get_unicode_simple_category(prev) == UnicodeSimpleCategory::Letter) {

while (ptr != end && (is_digit(*ptr) || *ptr == ' ' || *ptr == '-')) {

ptr++;

}

continue;

}

}

auto card_number_begin = ptr;

size_t digit_count = 0;

while (ptr != end && (is_digit(*ptr) || *ptr == ' ' || *ptr == '-')) {

if (*ptr == ' ' && digit_count >= 16 && digit_count <= 19 &&

digit_count == static_cast<size_t>(ptr - card_number_begin)) {

// continuous card number

break;

}

digit_count += static_cast<size_t>(is_digit(*ptr));

ptr++;

}

if (digit_count < 13 || digit_count > 19) {

continue;

}

auto card_number_end = ptr;

while (!is_digit(card_number_end[-1])) {

card_number_end--;

}

auto card_number_size = static_cast<size_t>(card_number_end - card_number_begin);

if (card_number_size > 2 * digit_count - 1) {

continue;

}

if (card_number_end != end) {

uint32 next;

next_utf8_unsafe(card_number_end, &next);

if (next == '-' || next == '_' || get_unicode_simple_category(next) == UnicodeSimpleCategory::Letter) {

continue;

}

}

result.emplace_back(card_number_begin, card_number_end);

}

return result;

}

static bool is_url_unicode_symbol(uint32 c) {

if (0x2000 <= c && c <= 0x206f) { // General Punctuation

// Zero Width Non-Joiner/Joiner and various dashes

return c == 0x200c || c == 0x200d || (0x2010 <= c && c <= 0x2015);

}

return get_unicode_simple_category(c) != UnicodeSimpleCategory::Separator;

}

static bool is_url_path_symbol(uint32 c) {

switch (c) {

case '\n':

case '<':

case '>':

case '"':

case 0xab: // «

case 0xbb: // »

return false;

default:

return is_url_unicode_symbol(c);

}

}

static vector<Slice> match_tg_urls(Slice str) {

vector<Slice> result;

const unsigned char *begin = str.ubegin();

const unsigned char *end = str.uend();

const unsigned char *ptr = begin;

// '(tg|ton|tonsite)://[a-z0-9_-]{1,253}([/?#][^\s\x{2000}-\x{200b}\x{200e}-\x{200f}\x{2016}-\x{206f}<>«»"]*)?'

Slice bad_path_end_chars(".:;,('?!`");

while (end - ptr > 5) {

ptr = static_cast<const unsigned char *>(std::memchr(ptr, ':', narrow_cast<int32>(end - ptr)));

if (ptr == nullptr) {

break;

}

const unsigned char *url_begin = nullptr;

if (end - ptr >= 3 && ptr[1] == '/' && ptr[2] == '/') {

if (ptr - begin >= 2 && to_lower(ptr[-2]) == 't' && to_lower(ptr[-1]) == 'g') {

url_begin = ptr - 2;

} else if (ptr - begin >= 3 && to_lower(ptr[-3]) == 't' && to_lower(ptr[-2]) == 'o' && to_lower(ptr[-1]) == 'n') {

url_begin = ptr - 3;

} else if (ptr - begin >= 7 && to_lower(ptr[-7]) == 't' && to_lower(ptr[-6]) == 'o' && to_lower(ptr[-5]) == 'n' &&

to_lower(ptr[-4]) == 's' && to_lower(ptr[-3]) == 'i' && to_lower(ptr[-2]) == 't' &&

to_lower(ptr[-1]) == 'e') {

url_begin = ptr - 3;

}

}

if (url_begin == nullptr) {

++ptr;

continue;

}

ptr += 3;

auto domain_begin = ptr;

while (ptr != end && ptr - domain_begin != 253 && is_alpha_digit_or_underscore_or_minus(*ptr)) {

ptr++;

}

if (ptr == domain_begin) {

continue;

}

if (ptr != end && (*ptr == '/' || *ptr == '?' || *ptr == '#')) {

auto path_end_ptr = ptr + 1;

while (path_end_ptr != end) {

uint32 code = 0;

auto next_ptr = next_utf8_unsafe(path_end_ptr, &code);

if (!is_url_path_symbol(code)) {

break;

}

path_end_ptr = next_ptr;

}

while (path_end_ptr > ptr + 1 && bad_path_end_chars.find(path_end_ptr[-1]) < bad_path_end_chars.size()) {

path_end_ptr--;

}

if (ptr[0] == '/' || path_end_ptr > ptr + 1) {

ptr = path_end_ptr;

}

}

result.emplace_back(url_begin, ptr);

}

return result;

}

static vector<Slice> match_urls(Slice str) {

vector<Slice> result;

const unsigned char *begin = str.ubegin();

const unsigned char *end = str.uend();

const auto &is_protocol_symbol = [](uint32 c) {

if (c < 0x80) {

// do not allow dots in the protocol

return is_alpha_digit(c) || c == '+' || c == '-';

}

// add unicode letters and digits to later discard protocol as invalid

return get_unicode_simple_category(c) != UnicodeSimpleCategory::Separator;

};

const auto &is_user_data_symbol = [](uint32 c) {

switch (c) {

case '\n':

case '/':

case '[':

case ']':

case '{':

case '}':

case '(':

case ')':

case '\'':

case '`':

case '<':

case '>':

case '"':

case '@':

case 0xab: // «

case 0xbb: // »

return false;

default:

return is_url_unicode_symbol(c);

}

};

const auto &is_domain_symbol = [](uint32 c) {

if (c < 0xc0) {

return c == '.' || is_alpha_digit_or_underscore_or_minus(c) || c == '~';

}

return is_url_unicode_symbol(c);

};

Slice bad_path_end_chars(".:;,('?!`");

while (true) {

auto dot_pos = str.find('.');

if (dot_pos > str.size() || dot_pos + 1 == str.size()) {

break;

}

if (str[dot_pos + 1] == ' ') {

// fast path

str = str.substr(dot_pos + 2);

begin = str.ubegin();

continue;

}

const unsigned char *domain_begin_ptr = begin + dot_pos;

while (domain_begin_ptr != begin) {

domain_begin_ptr = prev_utf8_unsafe(domain_begin_ptr);

uint32 code = 0;

auto next_ptr = next_utf8_unsafe(domain_begin_ptr, &code);

if (!is_domain_symbol(code)) {

domain_begin_ptr = next_ptr;

break;

}

}

const unsigned char *last_at_ptr = nullptr;

const unsigned char *domain_end_ptr = begin + dot_pos;

while (domain_end_ptr != end) {

uint32 code = 0;

auto next_ptr = next_utf8_unsafe(domain_end_ptr, &code);

if (code == '@') {

last_at_ptr = domain_end_ptr;

} else if (!is_domain_symbol(code)) {

break;

}

domain_end_ptr = next_ptr;

}

if (last_at_ptr != nullptr) {

while (domain_begin_ptr != begin) {

domain_begin_ptr = prev_utf8_unsafe(domain_begin_ptr);

uint32 code = 0;

auto next_ptr = next_utf8_unsafe(domain_begin_ptr, &code);

if (!is_user_data_symbol(code)) {

domain_begin_ptr = next_ptr;

break;

}

}

}

// LOG(ERROR) << "Domain: " << Slice(domain_begin_ptr, domain_end_ptr);

const unsigned char *url_end_ptr = domain_end_ptr;

if (url_end_ptr != end && url_end_ptr[0] == ':') {

auto port_end_ptr = url_end_ptr + 1;

while (port_end_ptr != end && is_digit(port_end_ptr[0])) {

port_end_ptr++;

}

auto port_begin_ptr = url_end_ptr + 1;

while (port_begin_ptr != port_end_ptr && *port_begin_ptr == '0') {

port_begin_ptr++;

}

if (port_begin_ptr != port_end_ptr && narrow_cast<int>(port_end_ptr - port_begin_ptr) <= 5 &&

to_integer<uint32>(Slice(port_begin_ptr, port_end_ptr)) <= 65535) {

url_end_ptr = port_end_ptr;

}

}

// LOG(ERROR) << "Domain_port: " << Slice(domain_begin_ptr, url_end_ptr);

if (url_end_ptr != end && (url_end_ptr[0] == '/' || url_end_ptr[0] == '?' || url_end_ptr[0] == '#')) {

auto path_end_ptr = url_end_ptr + 1;

while (path_end_ptr != end) {

uint32 code = 0;

auto next_ptr = next_utf8_unsafe(path_end_ptr, &code);

if (!is_url_path_symbol(code)) {

break;

}

path_end_ptr = next_ptr;

}

while (path_end_ptr > url_end_ptr + 1 && bad_path_end_chars.find(path_end_ptr[-1]) < bad_path_end_chars.size()) {

path_end_ptr--;

}

if (url_end_ptr[0] == '/' || path_end_ptr > url_end_ptr + 1) {

url_end_ptr = path_end_ptr;

}

}

while (url_end_ptr > begin + dot_pos + 1 && url_end_ptr[-1] == '.') {

url_end_ptr--;

}

// LOG(ERROR) << "Domain_port_path: " << Slice(domain_begin_ptr, url_end_ptr);

bool is_bad = false;

const unsigned char *url_begin_ptr = domain_begin_ptr;

if (url_begin_ptr != begin && url_begin_ptr[-1] == '@') {

if (last_at_ptr != nullptr) {

is_bad = true;

}

auto user_data_begin_ptr = url_begin_ptr - 1;

while (user_data_begin_ptr != begin) {

user_data_begin_ptr = prev_utf8_unsafe(user_data_begin_ptr);

uint32 code = 0;

auto next_ptr = next_utf8_unsafe(user_data_begin_ptr, &code);

if (!is_user_data_symbol(code)) {

user_data_begin_ptr = next_ptr;

break;

}

}

if (user_data_begin_ptr == url_begin_ptr - 1) {

is_bad = true;

}

url_begin_ptr = user_data_begin_ptr;

}

// LOG(ERROR) << "User_data_port_path: " << Slice(url_begin_ptr, url_end_ptr);

if (url_begin_ptr != begin) {

Slice prefix(begin, url_begin_ptr);

if (prefix.size() >= 6 && ends_with(prefix, "://")) {

auto protocol_begin_ptr = url_begin_ptr - 3;

while (protocol_begin_ptr != begin) {

protocol_begin_ptr = prev_utf8_unsafe(protocol_begin_ptr);

uint32 code = 0;

auto next_ptr = next_utf8_unsafe(protocol_begin_ptr, &code);

if (!is_protocol_symbol(code)) {

protocol_begin_ptr = next_ptr;

break;

}

}

auto protocol = to_lower(Slice(protocol_begin_ptr, url_begin_ptr - 3));

if (ends_with(protocol, "http") && protocol != "shttp") {

url_begin_ptr = url_begin_ptr - 7;

} else if (ends_with(protocol, "https")) {

url_begin_ptr = url_begin_ptr - 8;

} else if (ends_with(protocol, "ftp") && protocol != "tftp" && protocol != "sftp") {

url_begin_ptr = url_begin_ptr - 6;

} else if (ends_with(protocol, "tonsite")) {

url_begin_ptr = url_begin_ptr - 10;

} else {

is_bad = true;

}

} else {

auto prefix_end = prefix.uend();

auto prefix_back = prev_utf8_unsafe(prefix_end);

uint32 code = 0;

next_utf8_unsafe(prefix_back, &code);

if (is_word_character(code) || code == '/' || code == '#' || code == '@') {

is_bad = true;

}

}

}

// LOG(ERROR) << "Full: " << Slice(url_begin_ptr, url_end_ptr) << " " << is_bad;

if (!is_bad) {

if (url_end_ptr > begin + dot_pos + 1) {

result.emplace_back(url_begin_ptr, url_end_ptr);

}

while (url_end_ptr != end && url_end_ptr[0] == '.') {

url_end_ptr++;

}

} else {

while (url_end_ptr[-1] != '.') {

url_end_ptr--;

}

}

if (url_end_ptr <= begin + dot_pos) {

url_end_ptr = begin + dot_pos + 1;

}

str = str.substr(url_end_ptr - begin);

begin = url_end_ptr;

}

return result;

}

static bool is_valid_bank_card(Slice str) {

const size_t MIN_CARD_LENGTH = 13;

const size_t MAX_CARD_LENGTH = 19;