✊🏽 🙇🏼 🧑🏾‍🤝‍🧑🏻 Telegram on go: part 1, parsing the scheme 🚪 ⛷️ 🧑‍🤝‍🧑

The desire to write a high-quality client for my favorite messenger on go has been ripening for a long time, but only a month ago I decided that the time had come and I have sufficient qualifications for this.

Development is still in progress (and completely open source), but the fascinating path has already passed from a complete lack of understanding of the protocol to a relatively stable client. In a series of articles, I'll explain what challenges I faced and how I dealt with them. The techniques that I have applied can be useful when developing a client for any binary protocol with a schema.

Type Language

Let's start with Type Language or TL, a protocol description scheme. I will not delve into the description of the format, the Habré already has its analysis, I will tell you only briefly about it. It is somewhat similar to gRPC and describes the interaction scheme between the client and the server: a data structure and a set of methods.

Here is an example of a type description:

error#1fbadfee code:int32 message:string = Error;

Here 1fbadfee

this is the type id, error

its name, code and message are fields, and Error

this is the class name.

Methods are described in the same way, only instead of a type name there will be a method name, and instead of a class - a result type:

sendPM#3faceff text:string habrauser:string = Error;

This means that the method sendPM

takes arguments text

and habrauser

, and returns Error

, variants (constructors) which have been previously described, eg error#1fbadfee

.

, - . : ad-hoc, .. . participle, go, . ad-hoc .

, , , . : , , .

, . Definition

, :

func TestDefinition(t *testing.T) {
	for _, tt := range []struct {
		Case       string
		Input      string
		String     string
		Definition Definition
	}{
		{
			Case:  "inputPhoneCall",
			Input: "inputPhoneCall#1e36fded id:long access_hash:long = InputPhoneCall",
			Definition: Definition{
				ID:   0x1e36fded,
				Name: "inputPhoneCall",
				Params: []Parameter{
					{
						Name: "id",
						Type: bareLong,
					},
					{
						Name: "access_hash",
						Type: bareLong,
					},
				},
				Type: Type{Name: "InputPhoneCall"},
			},
		},
    // ...
  } {
		t.Run(tt.Case, func(t *testing.T) {
			var d Definition
			if err := d.Parse(tt.Input); err != nil {
				t.Fatal(err)
			}
			require.Equal(t, tt.Definition, d)
		})
  } 
}

, Flag

( , ), .

, , . :

	t.Run("Error", func(t *testing.T) {
		for _, invalid := range []string{
			"=0",
			"0 :{.0?InputFi00=0",
		} {
			t.Run(invalid, func(t *testing.T) {
				var d Definition
				if err := d.Parse(invalid); err == nil {
					t.Error("should error")
				}
			})
		}
	})

testdata

. _testdata

: , , go .

Sample.tl _testdata :

func TestParseSample(t *testing.T) {
	data, err := ioutil.ReadFile(filepath.Join("_testdata", "Sample.tl"))
	if err != nil {
		t.Fatal(err)
	}
	schema, err := Parse(bytes.NewReader(data))
	if err != nil {
		t.Fatal(err)
	}
  // ...
}

go , , filepath.Join

-.

(golden)

"golden files". , . , ( -update

). , . goldie .

func TestParser(t *testing.T) {
	for _, v := range []string{
		"td_api.tl",
		"telegram_api.tl",
		"telegram_api_header.tl",
		"layer.tl",
	} {
		t.Run(v, func(t *testing.T) {
			data, err := ioutil.ReadFile(filepath.Join("_testdata", v))
			if err != nil {
				t.Fatal(err)
			}
			schema, err := Parse(bytes.NewReader(data))
			if err != nil {
				t.Fatal(err)
			}
			t.Run("JSON", func(t *testing.T) {
				g := goldie.New(t,
					goldie.WithFixtureDir(filepath.Join("_golden", "parser", "json")),
					goldie.WithDiffEngine(goldie.ColoredDiff),
					goldie.WithNameSuffix(".json"),
				)
				g.AssertJson(t, v, schema)
			})
		})
	}
}

, json ( json). -update

, , _golden

.

(, json ) , .

Decode-Encode-Decode

, , decode-encode-decode, .

String() string

:

// Annotation represents an annotation comment, like //@name value.
type Annotation struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

func (a Annotation) String() string {
	var b strings.Builder
	b.WriteString("//")
	b.WriteRune('@')
	b.WriteString(a.Name)
	b.WriteRune(' ')
	b.WriteString(a.Value)
	return b.String()
}

, strings.Builder, String()

.

, , .

Fuzzing

() . , , (coverage-guided fuzzing). go go-fuzz . ( ) , . , syzkaller, go, Linux .

, , , , .

, Definition:

// +build fuzz

package tl

import "fmt"

func FuzzDefinition(data []byte) int {
	var d Definition
	if err := d.Parse(string(data)); err != nil {
		return 0
	}

	var other Definition
	if err := other.Parse(d.String()); err != nil {
		fmt.Printf("input: %s\n", string(data))
		fmt.Printf("parsed: %#v\n", d)
		panic(err)
	}

	return 1
}

, .

Decode-encode-decode-encode

We need to go deeper. :

(2)
(3)
(4) (2)

(4) (2) , .. - . , .

go-fuzz

Denial of Service , .. OOM. , go-fuzz , , .

corpus, , ( crashers, , , ). crashers , 0, . , , corpus , .

, go, , , .

, , , - . (STUN, TURN, SDP, MTProto, ...) .

, - . , , ( ) Telegram go:

( )
Network communication testing (unit, e2e)
Side effects testing (time, timeouts, PRNG)
CI, or set up the pipeline so that the Merge button is not scary to press

And I also want to say more thanks to the project participants who joined the project, without them it would be much more difficult.

Telegram on go: part 1, parsing the scheme