Redo heartbeats to save heart attack events

This commit is contained in:
jetsparrow 2021-04-22 17:41:35 +03:00
parent b0e1957ec8
commit 4131d2f22c
3 changed files with 47 additions and 32 deletions

View File

@ -56,18 +56,25 @@ namespace JetHerald.Controllers
}
}
[Route("api/heartbeat")]
[HttpGet]
public Task<IActionResult> HeartbeatGet(HeartbeatArgs args) => DoHeartbeat(args);
private async Task<IActionResult> DoHeartbeat(HeartbeatArgs args)
{
var heart = args.Heart ?? "General";
var t = await Db.GetTopic(args.Topic);
if (t == null)
return new NotFoundResult();
else if (!t.WriteToken.Equals(args.WriteToken, StringComparison.OrdinalIgnoreCase))
else if (!t.WriteToken.Equals(args.WriteToken, StringComparison.Ordinal))
return StatusCode(403);
if (t.ExpiryMessageSent)
await Herald.HeartbeatSent(t);
var affected = await Db.ReportHeartbeat(t.TopicId, heart, args.ExpiryTimeout);
await Db.AddExpiry(t.Name, args.ExpiryTimeout);
if (affected == 1)
await Herald.HeartbeatSent(t);
return new OkResult();
}
@ -75,6 +82,7 @@ namespace JetHerald.Controllers
public class HeartbeatArgs
{
[JsonPropertyName("Topic")] public string Topic;
[JsonPropertyName("Heart")] public string Heart;
[JsonPropertyName("ExpiryTimeout")] public int ExpiryTimeout;
[JsonPropertyName("WriteToken")] public string WriteToken;
}

View File

@ -28,11 +28,14 @@ namespace JetHerald
=> Name == Description ? Name : $"{Name}: {Description}";
}
public class ExpiredTopicChat
public class HeartAttack
{
public NamespacedId Chat;
public string Description;
public ulong HeartattackId { get; set; }
public uint TopicId { get; set; }
public string Heart { get; set; }
public DateTime ExpiryTime { get; set; }
public string Description { get; set; }
}
public async Task<int> DeleteTopic(string name, string adminToken)
@ -131,38 +134,37 @@ namespace JetHerald
new { topicName, chat });
}
public Task AddExpiry(string topicName, int addedTime)
public async Task<int> ReportHeartbeat(uint topicId, string heart, int timeoutSeconds)
{
using var c = GetConnection();
return c.ExecuteAsync(
" UPDATE topic" +
" SET ExpiryTime = CURRENT_TIMESTAMP() + INTERVAL @addedTime SECOND," +
" ExpiryMessageSent = 0" +
" WHERE Name = @topicName",
new { topicName, addedTime });
return await c.ExecuteAsync(
@"
INSERT INTO heartbeat
(TopicId, Heart, ExpiryTime)
VALUES
(@topicId, @heart, @expiry)
ON DUPLICATE KEY UPDATE
ExpiryTime = @expiry;
",
new { topicId, heart, expiry = DateTime.UtcNow.AddSeconds(timeoutSeconds)});
}
public Task<IEnumerable<ExpiredTopicChat>> GetExpiredTopics(CancellationToken token = default)
public async Task<IEnumerable<HeartAttack>> ProcessHeartAttacks()
{
using var c = GetConnection();
return c.QueryAsync<ExpiredTopicChat>(
" SELECT tc.Chat, t.Description, t.ExpiryTime" +
" FROM topic_chat tc" +
" INNER JOIN topic t ON t.TopicId = tc.TopicId" +
" WHERE t.ExpiryTime < CURRENT_TIMESTAMP() AND NOT t.ExpiryMessageSent",
token);
return await c.QueryAsync<HeartAttack>("CALL process_heartattacks();");
}
public Task MarkExpiredTopics(CancellationToken token = default)
public async Task MarkHeartAttackReported(ulong id)
{
using var c = GetConnection();
return c.ExecuteAsync(
" UPDATE topic t" +
" SET t.ExpiryMessageSent = 1" +
" WHERE t.ExpiryTime < CURRENT_TIMESTAMP()",
token);
await c.ExecuteAsync("UPDATE heartattack SET Reported = 1 WHERE HeartattackId = @id", new {id});
}
public Db(IOptions<Options.ConnectionStrings> cfg)
{
Config = cfg.Value;

View File

@ -88,13 +88,18 @@ namespace JetHerald
while (!token.IsCancellationRequested)
{
await Task.Delay(1000 * 10, token);
try
{
foreach (var chatSent in await Db.GetExpiredTopics(token))
await SendMessageImpl(chatSent.Chat, $"!{chatSent.Description}!:\nTimeout expired at {chatSent.ExpiryTime}");
await Db.MarkExpiredTopics(token);
var attacks = await Db.ProcessHeartAttacks();
foreach (var attack in attacks)
{
var chats = await Db.GetChatsForTopic(attack.TopicId);
foreach (var chat in chats)
await SendMessageImpl(chat, $"!{attack.Description}!:\nTimeout expired at {attack.ExpiryTime}");
await Db.MarkHeartAttackReported(attack.HeartattackId);
if (token.IsCancellationRequested)
return;
}
}
catch (Exception e)
{