Skip to content

Request limiter - prevent hitting Telegram's API limits #397

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Feb 20, 2017
Merged
3 changes: 3 additions & 0 deletions examples/cron.php
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@
//$telegram->setDownloadPath('../Download');
//$telegram->setUploadPath('../Upload');

// Requests Limiter (tries to prevent reaching Telegram API limits)
$telegram->enableLimiter();

// Run user selected commands
$telegram->runCommands($commands);
} catch (Longman\TelegramBot\Exception\TelegramException $e) {
Expand Down
3 changes: 3 additions & 0 deletions examples/getUpdatesCLI.php
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@
//$telegram->enableBotan('your_token');
//$telegram->enableBotan('your_token', ['timeout' => 3]);

// Requests Limiter (tries to prevent reaching Telegram API limits)
$telegram->enableLimiter();

// Handle telegram getUpdates request
$serverResponse = $telegram->handleGetUpdates();

Expand Down
3 changes: 3 additions & 0 deletions examples/hook.php
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,9 @@
//$telegram->enableBotan('your_token');
//$telegram->enableBotan('your_token', ['timeout' => 3]);

// Requests Limiter (tries to prevent reaching Telegram API limits)
$telegram->enableLimiter();

// Handle telegram webhook request
$telegram->handle();
} catch (Longman\TelegramBot\Exception\TelegramException $e) {
Expand Down
80 changes: 80 additions & 0 deletions src/DB.php
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ protected static function defineTables()
'edited_message',
'inline_query',
'message',
'request_limiter',
'telegram_update',
'user',
'user_chat',
Expand Down Expand Up @@ -1065,4 +1066,83 @@ public static function selectChats(
throw new TelegramException($e->getMessage());
}
}

/**
* Get Telegram API request count for current chat / message
*
* @param integer $chat_id
* @param string $inline_message_id
*
* @return array|bool (Array containing TOTAL and CURRENT fields or false on invalid arguments)
* @throws \Longman\TelegramBot\Exception\TelegramException
*/
public static function getTelegramRequestCount($chat_id = null, $inline_message_id = null)
{
if (!self::isDbConnected()) {
return false;
}

try {
$sth = self::$pdo->prepare('SELECT
(SELECT COUNT(*) FROM `' . TB_REQUEST_LIMITER . '` WHERE `created_at` >= :date) as LIMIT_PER_SEC_ALL,
(SELECT COUNT(*) FROM `' . TB_REQUEST_LIMITER . '` WHERE ((`chat_id` = :chat_id AND `inline_message_id` IS NULL) OR (`inline_message_id` = :inline_message_id AND `chat_id` IS NULL)) AND `created_at` >= :date) as LIMIT_PER_SEC,
(SELECT COUNT(*) FROM `' . TB_REQUEST_LIMITER . '` WHERE `chat_id` = :chat_id AND `created_at` >= :date_minute) as LIMIT_PER_MINUTE
');

$date = self::getTimestamp(time());
$date_minute = self::getTimestamp(strtotime('-1 minute'));

$sth->bindParam(':chat_id', $chat_id, \PDO::PARAM_STR);
$sth->bindParam(':inline_message_id', $inline_message_id, \PDO::PARAM_STR);
$sth->bindParam(':date', $date, \PDO::PARAM_STR);
$sth->bindParam(':date_minute', $date_minute, \PDO::PARAM_STR);

$sth->execute();

return $sth->fetch();
} catch (\Exception $e) {
throw new TelegramException($e->getMessage());
}
}

/**
* Insert Telegram API request in db
*
* @param string $method
* @param array $data
*
* @return bool If the insert was successful
* @throws \Longman\TelegramBot\Exception\TelegramException
*/
public static function insertTelegramRequest($method, $data)
{
if (!self::isDbConnected()) {
return false;
}

$chat_id = ((isset($data['chat_id'])) ? $data['chat_id'] : null);
$inline_message_id = (isset($data['inline_message_id']) ? $data['inline_message_id'] : null);

try {
$sth = self::$pdo->prepare('INSERT INTO `' . TB_REQUEST_LIMITER . '`
(
`method`, `chat_id`, `inline_message_id`, `created_at`
)
VALUES (
:method, :chat_id, :inline_message_id, :date
);
');

$created_at = self::getTimestamp();

$sth->bindParam(':chat_id', $chat_id, \PDO::PARAM_STR);
$sth->bindParam(':inline_message_id', $inline_message_id, \PDO::PARAM_STR);
$sth->bindParam(':method', $method, \PDO::PARAM_STR);
$sth->bindParam(':date', $created_at, \PDO::PARAM_STR);

return $sth->execute();
} catch (\Exception $e) {
throw new TelegramException($e->getMessage());
}
}
}
84 changes: 84 additions & 0 deletions src/Request.php
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,13 @@ class Request
*/
private static $input;

/**
* Request limiter
*
* @var boolean
*/
private static $limiter_enabled;

/**
* Available actions to send
*
Expand Down Expand Up @@ -318,6 +325,8 @@ public static function send($action, array $data = [])

self::ensureNonEmptyData($data);

self::limitTelegramRequests($action, $data);

$response = json_decode(self::execute($action, $data), true);

if (null === $response) {
Expand Down Expand Up @@ -976,4 +985,79 @@ public static function getWebhookInfo()
// Must send some arbitrary data for this to work for now...
return self::send('getWebhookInfo', ['info']);
}

/**
* Enable request limiter
*
* @param boolean $value
*/
public static function setLimiter($value = true)
{
if (DB::isDbConnected()) {
self::$limiter_enabled = $value;
}
}

/**
* This functions delays API requests to prevent reaching Telegram API limits
* Can be disabled while in execution by 'Request::setLimiter(false)'
*
* @link https://core.telegram.org/bots/faq#my-bot-is-hitting-limits-how-do-i-avoid-this
*
* @param string $action
* @param array $data
*
* @throws \Longman\TelegramBot\Exception\TelegramException
*/
private static function limitTelegramRequests($action, array $data = [])
{
if (self::$limiter_enabled) {
$limited_methods = [
'sendMessage',
'forwardMessage',
'sendPhoto',
'sendAudio',
'sendDocument',
'sendSticker',
'sendVideo',
'sendVoice',
'sendLocation',
'sendVenue',
'sendContact',
'editMessageText',
'editMessageCaption',
'editMessageReplyMarkup',
];

$chat_id = isset($data['chat_id']) ? $data['chat_id'] : null;
$inline_message_id = isset($data['inline_message_id']) ? $data['inline_message_id'] : null;

if (($chat_id || $inline_message_id) && in_array($action, $limited_methods)) {
$timeout = 60;

while (true) {
if ($timeout <= 0) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You could also do $timeout-- here instead of $timeout = $timeout - 1; further down.

throw new TelegramException('Timed out while waiting for a request spot!');
}

$requests = DB::getTelegramRequestCount($chat_id, $inline_message_id);

if (
$requests['LIMIT_PER_SEC'] == 0 && // No more than one message per second inside a particular chat
(
(($chat_id > 0 || $inline_message_id) && $requests['LIMIT_PER_SEC_ALL'] < 30) || // No more than 30 messages per second globally
($chat_id < 0 && $requests['LIMIT_PER_MINUTE'] < 20) // No more than 20 messages per minute for group chats
)
) {
break;
}

$timeout--;
sleep(1);
}

DB::insertTelegramRequest($action, $data);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I noticed that entries once added, never get deleted from the DB once handled.
Shouldn't they be cleared again after being processed?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But if I remove them next requests will think they are ok to go.
I could add a query to remove entries older than 1 minute but I don't think it will be good practice to run such query every second (or more under heavy load)...?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, I'm just thinking long term here. As this is a temporary table, it just gets bigger and bigger, in some cases pretty quick, right?
So some sort of cleanup would be good I think.

Maybe something like:

if (mt_rand(0, 999) === 42) {
    do_cleanup();
}

Or maybe I'm going too far here and this isn't really a problem...

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess, in my case I'm cleaning the table every 5 minutes, set as a task in Mysql.

I have no 'good' idea how to implement cleanup inside this code without adding potential delays or performance losses.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, then let's leave this as is for the moment and passively think of something 😉

}
}
}
}
10 changes: 10 additions & 0 deletions src/Telegram.php
Original file line number Diff line number Diff line change
Expand Up @@ -841,6 +841,16 @@ public function enableBotan($token, array $options = [])
return $this;
}

/**
* Enable requests limiter
*/
public function enableLimiter()
{
Request::setLimiter(true);

return $this;
}

/**
* Run provided commands
*
Expand Down
10 changes: 10 additions & 0 deletions structure.sql
Original file line number Diff line number Diff line change
Expand Up @@ -212,3 +212,13 @@ CREATE TABLE IF NOT EXISTS `botan_shortener` (

FOREIGN KEY (`user_id`) REFERENCES `user` (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;

CREATE TABLE IF NOT EXISTS `request_limiter` (
`id` bigint UNSIGNED AUTO_INCREMENT COMMENT 'Unique identifier for this entry',
`chat_id` char(255) NULL DEFAULT NULL COMMENT 'Unique chat identifier',
`inline_message_id` char(255) NULL DEFAULT NULL COMMENT 'Identifier of the sent inline message',
`method` char(255) DEFAULT NULL COMMENT 'Request method',
`created_at` timestamp NULL DEFAULT NULL COMMENT 'Entry date creation',

PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT charSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci;