• General introduction
• FP
  
  
  • Introduction to FP
  • Fundamental principles of FP
  • Basic terms
  • Built-in FP behavior in Python
  • Xoltar Toolkit Library
  • Returns library
  • Literature
• Generators
  
  
  • Introduction to iterators
  • Introduction to generators
  • Generators vs iterators
  • Generators as a pipeline
  • The yield from concept
  • Data routing on generators (multiplexing, broadcasting)
  • Generator tracing example
  • Standard tools generators
  • conclusions
    
    
    • pros
    • Minuses
  • Literature
• Outcome

General introduction

Python, , , . — . , , , Python. , , , . "Fluent Python", , , .

, , . — , , .

: .

“?”. , , . , .

“?”. , , , , , . .

- “ ” “ ”.

– / / . , “ ”. , C, , .

- () , , . : , , , . , C#, Java.

– , ( ). , – Haskell, Lisp.

, . , , , , .

• (First Class Object).
  
  , , — , ..
• . , .
• (lists, Lisp — LISt Processing). .
• (High Order Functions). – , .
  
  

  . Python , map. Iterable , Iterable Iterator , .
  
  

• “” (Pure Functions) – .. ( : -).
  
  

  Python . , .
  
  

• , , , .

, ( )

• .

  
  
  

  — , . .

  
  
  

• 
  
  

  • .

    
    
    

    .

    
    
    

  • .

    
    
    

    - , . , .

    
    
    

  • .

    
    
    

    .
    
    .

    
    
    
  
  
  

• —

  
  
  

• .

  
  
  

  , - , . — , . , , , , , .

  
  
  

• .

  
  
  

  — , . .

  
  
  

  • .

    
    
    

    , , , . .

    
    
    

    , , , . , , .

    
    
    
  
  
  

• (closure)

  
  
  

  — . © Steve Majewski
  
  — , .

  
  
  

Python

Python — map(), reduce(), filter() lambda. Python 1.x apply(), , . Python 2.0 . Python 2.3 , Python 3.0

, Python; , (if, elif, else, assert, try, except, finally, for, break, continue, while, def) , . , , , " Python" ( , Lisp'), , .

, , . if/elif/else Python

# Normal statement-based flow control
if <cond1>: 
    func1() 
elif <cond2>: 
    func2() 
else: 
    func3() 

    # Equivalent "short circuit" expression
(<cond1> and func1()) or (<cond2> and func2()) or (func3()) 

. skymorp, , func1, func2 () func3 non falsy . , func , (func3) .

lambda

pr = lambda s:s 
namenum = lambda x: (x==1 and pr("one")) or (x==2 and pr("two")) or (pr("other"))
assert namenum(1) == 'one' 
assert namenum(2) == 'two' 
assert namenum(3) == 'other'

, . for map().

for e in lst:  
    func(e)      # statement-based loop

map(func,lst)    # map-based loop

.

do_it = lambda f: f()

# let f1, f2, f3 (etc) be functions that perform actions

map(do_it, [f1,f2,f3])

while , .

# statement-based while loop
while <cond>: 
    <pre-suite> 
    if <break_condition>: 
        break
    else: 
        <suite> 

# FP-style recursive while loop
def while_block(): 
    <pre-suite> 
    if <break_condition>: 
        return 1 
    else: 
        <suite> 
        return 0 

while_FP = lambda: (<cond> and while_block()) or while_FP() 
while_FP()

while while_block(), , (statements). (, , if/else ).

, ( while myvar == 7) , ( ) - ( while_block()). — while_block() .

:

# imperative version of "echo()"
def echo_IMP():
    while 1: 
        x = input("IMP -- ") 
        if x == 'quit': 
            break
        else:
            print(x) 

echo_IMP() 

# utility function for "identity with side-effect"
def monadic_print(x):
    print(x) 
    return x 
    # FP version of "echo()" 

echo_FP = lambda: monadic_print(input("FP -- ")) == 'quit' or echo_FP() 
echo_FP()

. skymorp, , print , input("IMP -- ") == 'quit'. print .

, , /, ( — , ).

monadic_print(), , . , , monadic_print(x) , x.

, — "?!". , , Python. (, , ) — , , . , , - , .

, .

# Nested loop procedural style for finding big products 
xs = (1,2,3,4) 
ys = (10,15,3,22) 
bigmuls = [] 
# ...more stuff...
for x in xs: 
    for y in ys: 
        # ...more stuff...
        if x*y > 25: 
            bigmuls.append((x,y)) 
        # ...more stuff...
    # ...more stuff...
    print(bigmuls)

, #...more stuff... — , .

xs, ys, bigmuls, x, y . , , , .

, / , . (del) .

. , . :

bigmuls = lambda xs,ys: filter(lambda (x,y):x*y > 25, combine(xs,ys))
combine = lambda xs,ys: map(None, xs*len(ys), dupelms(ys,len(xs)))
dupelms = lambda lst,n: reduce(lambda s,t:s+t, map(lambda l,n=n: [l]*n, lst))
print(bigmuls((1,2,3,4),(10,15,3,22)))

, . - ( ) . , , . — — ( ) :

print([(x,y) for x in (1,2,3,4) for y in (10,15,3,22) if x*y > 25])

, , list, tuple, set, dict comprehensions generator expressions — , , , -

Xoltar Toolkit

, Python 2, . Xoltar Toolkit (Bryn Keller) .

Python. functional, Xoltar Toolkit lazy, , " ". , Xoltar Toolkit , Haskell.

Python , . , , . , .

>>> car = lambda lst: lst[0] 
>>> cdr = lambda lst: lst[1:] 
>>> sum2 = lambda lst: car(lst)+car(cdr(lst)) 
>>> sum2(range(10))
1 
>>> car = lambda lst: lst[2] 
>>> sum2(range(10))
5

, sum2(range(10)) , , .

, functional Bindings, .

>>> from functional import * 
>>> let = Bindings() 
>>> let.car = lambda lst: lst[0] 
>>> let.car = lambda lst: lst[2] 
Traceback (innermost last): 
    File "<stdin>", 
        line 1, in ? File "d:\tools\functional.py", 
        line 976, in __setattr__ raise BindingError, "Binding '%s' cannot be modified." % name 
        functional.BindingError: Binding 'car' cannot be modified. >>> car(range(10)) 0

, BindingError, .

returns

, , , Python. maybe

from returns.maybe import Maybe, maybe
@maybe  # decorator to convert existing Optional[int] to Maybe[int]
def bad_function() -> Optional[int]:
    ...
    maybe_number: Maybe[float] = bad_function().map(
    lambda number: number / 2,
    )
# => Maybe will return Some[float] only if there's a non-None value
#    Otherwise, will return Nothing

, :

# Imperative style
user: Optional[User]
discount_program: Optional['DiscountProgram'] = None
if user is not None:
     balance = user.get_balance()
     if balance is not None:
         credit = balance.credit_amount()
         if credit is not None and credit > 0:
            discount_program = choose_discount(credit)

# same with returns

user: Optional[User]
# Type hint here is optional, it only helps the reader here:
discount_program: Maybe['DiscountProgram'] = Maybe.from_value(
    user,
    ).map(  # This won't be called if `user is None`
    lambda real_user: real_user.get_balance(),
    ).map(  # This won't be called if `real_user.get_balance()` returns None
    lambda balance: balance.credit_amount(),
    ).map(  # And so on!
    lambda credit: choose_discount(credit) if credit > 0 else None,
    )

, ,

# Imperative style
def fetch_user_profile(user_id: int) -> 'UserProfile':
    """Fetches UserProfile dict from foreign API."""
    response = requests.get('/api/users/{0}'.format(user_id))
    # What if we try to find user that does not exist?
    # Or network will go down? Or the server will return 500?
    # In this case the next line will fail with an exception.
    # We need to handle all possible errors in this function
    # and do not return corrupt data to consumers.
    response.raise_for_status()
    # What if we have received invalid JSON?
    # Next line will raise an exception!
    return response.json()

, returns

import requests
from returns.result import Result, safe
from returns.pipeline import flow
from returns.pointfree import bind
def fetch_user_profile(user_id: int) -> Result['UserProfile', Exception]:
    """Fetches `UserProfile` TypedDict from foreign API."""
    return flow(
        user_id,
        _make_request,
        bind(_parse_json),
    )

@safe
def _make_request(user_id: int) -> requests.Response:
    response = requests.get('/api/users/{0}'.format(user_id))
    response.raise_for_status()
    return response

@safe
def _parse_json(response: requests.Response) -> 'UserProfile':
    return response.json()

, : , , , , .

, , @safe. Success [YourType] Failure [Exception]. !

, .

. , , returns , - , .

• https://devpractice.ru/fp-python-part1-general/
• Python
  
  
  • https://www.ibm.com/developerworks/ru/library/l-prog/index.html
  • https://www.ibm.com/developerworks/ru/library/l-prog2/index.html
• https://degoes.net/articles/fp-glossary
• https://returns.readthedocs.io/en/latest/

— . , .

>>> for x in [1,4,5,10]:
... print(x, end=' ')
...
1 4 5 10

, , ( ). , —

>>> items = [1, 4, 5]
>>> it = iter(items)
>>> it.__next__()
1
>>> it.__next__()
4
>>> it.__next__()
5
>>> it.__next__()

.

for x in obj:
    # statements

:

_iter = iter(obj) # Get iterator object
while 1:
    try:
        x = _iter.__next__() # Get next item
    except StopIteration: # No more items
        break
    # statements

, , iter() . , __iter__() __next__().

, , :

>>> for x in Countdown(10):
... print(x, end=' ')
...
10 9 8 7 6 5 4 3 2 1

:

class Countdown(object):
    def __init__(self,start):
        self.start = start
    def __iter__(self):
        return CountdownIter(self.start)

class CountdownIter(object):
    def __init__(self, count):
        self.count = count
    def __next__(self):
        if self.count <= 0:
            raise StopIteration
        r = self.count
        self.count -= 1
        return r

— ,

def countdown(n):
    while n > 0:
        yield n
        n -= 1

, , ( yield). -. .

def countdown(n):
    print("Counting down from", n)
    while n > 0:
        yield n
        n -= 1
>>> x = countdown(10)
>>> x
<generator object at 0x58490>
>>>

__next__().

>>> x = countdown(10)
>>> x
<generator object at 0x58490>
>>> x.__next__()
Counting down from 10
10
>>>

yield , . __next__(). StopIteration.

>>> x.__next__()
9
>>> x.__next__()
8
>>>
...
>>> x.__next__()
1
>>> x.__next__()
Traceback (most recent call last):
    File "<stdin>", line 1, in ?
        StopIteration
>>>

:

• —
• (__next__, __iter__ . .), .. yield Python .

>>> def x():
...     return 1
... 
>>> def y():
...     yield 1
... 
>>> [i for i in dir(y()) if i not in dir(x())]
['__del__', '__iter__', '__name__', '__next__', '__qualname__', 'close', 'gi_code', 'gi_frame', 'gi_running', 'gi_yieldfrom', 'send', 'throw']

, generator object.

>>> a = [1,2,3,4]
>>> b = (2*x for x in a)
>>> b
<generator object at 0x58760>
>>> for i in b: print(b, end=' ')
...
2 4 6 8

(expression for i in s if condition)
# the same with
for i in s:
    if condition:
        yield expression

vs

, . — . , , . , , ( , )

, . , :

, , - Apache. ,

:

81.107.39.38 - ... "GET /ply/ply.html HTTP/1.1" 200 97238

:

bytes_sent = line.rsplit(None,1)[1]

.

81.107.39.38 - ... "GET /ply/ HTTP/1.1" 304 -

if bytes_sent != '-':
    bytes_sent = int(bytes_sent)

,

with open("access-log") as wwwlog:
    total = 0
    for line in wwwlog:
        bytes_sent = line.rsplit(None,1)[1]
        if bytes_sent != '-':
            total += int(bytes_sent)
    print("Total", total)

. . , .

with open("access-log") as wwwlog:
    bytecolumn = (line.rsplit(None,1)[1] for line in wwwlog)
    bytes_sent = (int(x) for x in bytecolumn if x != '-')
    print("Total", sum(bytes_sent))

, ,

, . , , . .

, . 1.3 18.6 , 16,7 .

AWK , 70.5

awk '{ total += $NF } END { print total }' big-access-log

:

• , 10% ,
• , ,
• , ,

. , ? , , .

yield from

'yield from'

def countdown(n):
    while n > 0:
        yield n
        n -= 1

def countup(stop):
    n = 1
    while n < stop:
        yield n
        n += 1

def up_and_down(n):
    yield from countup(n)
    yield from countdown(n)

>>> for x in up_and_down(3):
... print(x)
...
1
2
3
2
1
>>>

, python (3.5 ) yield from await, await , .. await — , . yield from — await.

(, )

— ,

, ( ) ( ). , .

# same with `tail -f`

def follow(thefile):
    thefile.seek(0, os.SEEK_END) # End-of-file
    while True:
        line = thefile.readline()
        if not line:
            time.sleep(0.1) # Sleep briefly
            continue
        yield line

def gen_cat(sources):
    #           
    for src in sources:
        yield from src

def genfrom_queue(thequeue):
    while True:
        item = thequeue.get()
        if item is StopIteration:
            break
        yield item

def sendto_queue(source, thequeue):
    for item in source:
        thequeue.put(item)
    thequeue.put(StopIteration)

def multiplex(sources):
    in_q = queue.Queue()
    consumers = []
    for src in sources:
        thr = threading.Thread(target=sendto_queue, args=(src, in_q))
        thr.start()
        consumers.append(genfrom_queue(in_q))
    return gen_cat(consumers)

def broadcast(source, consumers):
    for item in source:
        for c in consumers:
            c.send(item)

class Consumer(object):
    def send(self,item):
        print(self, "got", item)

if __name__ == '__main__':
    c1 = Consumer()
    c2 = Consumer()
    c3 = Consumer()

    log1 = follow(open("foo/access-log"))
    log2 = follow(open("bar/access-log"))
    log3 = follow(open("baz/access-log"))

    lines = multiplex([log1, log2, log3])

    broadcast(lines,[c1,c2,c3])

— , .

, , , , .

def trace(source):
    for item in source:
        print(item)
        yield item

lines = follow(open("access-log"))
log = trace(apache_log(lines))
r404 = trace(r for r in log if r['status'] == 404)

— , , , ,

. 3.0 . , pathlib.Path.rglob, glob.iglob, os.walk, range, map, filter. — itertools.

:

• —
• ,
• ,
• ,
• (, , )

• , ,
• , .

• https://www.dabeaz.com/generators/
• https://www.dabeaz.com/generators/Generators.pdf ( )
• https://wiki.python.org/moin/Generators (. Links)
• https://www.oreilly.com/library/view/fluent-python/9781491946237/

, . , Python, , - Python . , , .

, , . , — , Python.

Our main task is to write clear, understandable, beautiful, testable code and choose the right tools for this. FP is not an end in itself, but only a means, as always, to be able to write even better code!

If you find errors, write in the telegram Niccolumor email lastsal@mail.ru. I would be glad to receive constructive criticism.