Is it really impossible to do without kafokas and rabbits when you receive 10,000 events per second?

I once hosted a webinar on how to receive 10,000 events per second. He showed this picture, the audience saw a lilac layer, and it began: "Guys, why do we need all these kafkas and rabbit, can we really not do without them?" We answered: "Why, why, to pass the social security!" 

Very funny, but let me explain it anyway.

---

.

,

, , . Kafka, Rabbit . , . 

: 10 , — , — 50 . , . 2 , 10 . , . — .

— -, . 

, . . : , . http- — 5, 10 — , . 

. « », : « , , ». «» , .

— — , . 

, , . , , , e-mail . , , , .

— — -, .

, . ? , . , . - , — (, ). , .

---

, .

-. . .

resource "yandex_compute_instance_group" "events-api-ig" {
  name               = "events-api-ig"
  service_account_id = yandex_iam_service_account.instances.id
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

. CPU, , ..

instance_template {
    platform_id = "standard-v2"
    resources {
      memory = 2
      cores  = 2
    }
    boot_disk {
      mode = "READ_WRITE"
      initialize_params {
        image_id = data.yandex_compute_image.container-optimized-image.id
        size = 10
      }
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

, .

}
    network_interface {
      network_id = yandex_vpc_network.internal.id
      subnet_ids = [yandex_vpc_subnet.internal-a.id, yandex_vpc_subnet.internal-b.id, yandex_vpc_subnet.internal-c.id]
      nat = true
    }
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

— scale_policy.

fixed scale A, B, C.

scale_policy {
    fixed_scale {
      size = 3
    }
  }

  allocation_policy {
    zones = ["ru-central1-a", "ru-central1-b", "ru-central1-c"]
  }
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

auto_scale — .

scale_policy {
auto_scale {
    initial_size = 3
    measurment_duration = 60
    cpu_utilization_target = 60
    min_zone_size = 1
    max_size = 6
    warmup_duration = 60
    stabilization_duration = 180
}
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

, , — cpu utilization target. , . .

, .

- load-. , 84.201.147.84 80, - — 8080. 

, Yandex.Tank . 20 5 . 

---

, .

(A, B C), , . .

, . , .

. , , ( ) 23 , 12,8 . - . , .

— .

, , CPU . : overload.yandex.net/256194.

---

golang. .

package main

import (
    "encoding/json"
    "flag"
    "io"
    "io/ioutil"
    "log"
    "net/http"
    "strings"

)
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

github.com/Shopify/sarama — .

github.com/prometheus/client_golang/prometheus, API Metrics.

github.com/streadway/amqp rabbitmq.

, .

var (
    // Config options
    addr     = flag.String("addr", ":8080", "TCP address to listen to")
    kafka    = flag.String("kafka", "127.0.0.1:9092", "Kafka endpoints")
    enableKafka    = flag.Bool("enable-kafka", false, "Enable Kafka or not")
amqp    = flag.String("amqp", "amqp://guest:guest@127.0.0.1:5672/", "AMQP URI")
enableAmqp    = flag.Bool("enable-amqp", false, "Enable AMQP or not")
sqsUri    = flag.String("sqs-uri", "", "SQS URI")
sqsId    = flag.String("sqs-id", "", SQS Access id")
sqsSecret    = flag.String("sqs-secret", "", "SQS Secret key")
enableSqs    = flag.Bool("enable-sqs", false, "Enable SQS or not")

    

    // Declaring prometheus metrics
    apiDurations = prometheus.NewSummary(
        prometheus.SummaryOpts{
            Name:       "api_durations_seconds",
            Help:       "API duration seconds",
            Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
        },
    )
      
      

        
        
        
      

    
        
        
        
      
      

        
        
        
      

    
    

— ().

— .

.

— .

— amqp .

— sqs ..

. prometheus .

main , Load.

sqs, ..

http :

/status okey, load-, .

/post/kafka, . /post/amqp /post/sqs.

— , . , .

- . , SSD ( , ), zookeeper. 200 ! — « », . 120 .

, — CPU, , - . , .

: topic, , . topic 50 partitions. .

, topic load 3 . Partition 1 Kafka 1, — 2, — 3. . , , , . .

50 , 50 1 — 50 . .

zookeeper. 3 . , , 2 . 2 2 . , : «, », 2 .

Terraform

terraform-, kafka.tf .

3 : resource “yandex compute instance” “zookeeper count = 3”.

“zookeeper_deploy”, . , , . . ansible .

, , .

RabbitMQ 

, , . , ! , .

— exchanges , delayed messages, deadletter . . , . — . , , . — .

. , , .

— RabbitMQ. — , , .. — kafka. .

: .