Withdraw!

The next transition is a withdrawal. Naively, we add another command:

open(_Data) ->
    [
        {open, {call, bank_statem, deposit, [pos_integer()]}},
        {open, {call, bank_statem, withdraw, [pos_integer()]}}
    ].

Which causes an immediate failure:

===> Testing prop_bank_statem:prop_test()
...
=ERROR REPORT==== 29-Apr-2018::15:08:41 ===
** State machine bank_statem terminating
** Last event = {{call,{,#Ref}},
                 {withdraw,2}}
** When server state  = {open,#{balance => 1}}
** Reason for termination = error:function_clause
** Callback mode = state_functions
** Stacktrace =
**  [{bank_statem,open,
                  [{call,{,#Ref}},
                   {withdraw,2},
                   #{balance => 1}],
                  [{file,"/app/_build/test/lib/bank_statem/src/bank_statem.erl"},
                   {line,46}]},
...
Crash dump is being written to: erl_crash.dump...done

Having the entire process crash isn’t very useful, so we follow the recommendation to use the TRAPEXIT macro:

prop_test() ->
    ?FORALL(Cmds, proper_fsm:commands(?MODULE),
        ?TRAPEXIT(
            begin
                bank_statem:start_link(),
                {History,State,Result} = proper_fsm:run_commands(?MODULE, Cmds),
                bank_statem:stop(),
                ?WHENFAIL(io:format("History: ~p\nState: ~p\nResult: ~p\n", [History,State,Result]),
                    aggregate(zip(proper_fsm:state_names(History), command_names(Cmds)), Result =:= ok))
            end)).

This allows shrinking to take place, and gives us (slightly) more useful output:

===> Testing prop_bank_statem:prop_test()
.!
Failed: After 2 test(s).
A linked process died with reason {function_clause,[{bank_statem,open,[{call,{,#Ref}},{withdraw,3},#{balance=>0}],[{file,[47,97,112,112,47,95,98,117,105,108,100,47,116,101,115,116,47,108,105,98,47,98,97,110,107,95,115,116,97,116,101,109,47,115,114,99,47,98,97,110,107,95,115,116,97,116,101,109,46,101,114,108]},{line,46}]},{gen_statem,call_state_function,5,[{file,[103,101,110,95,115,116,97,116,101,109,46,101,114,108]},{line,1633}]},{gen_statem,loop_event_state_function,6,[{file,[103,101,110,95,115,116,97,116,101,109,46,101,114,108]},{line,1023}]},{proc_lib,init_p_do_apply,3,[{file,[112,114,111,99,95,108,105,98,46,101,114,108]},{line,247}]}]}.

=ERROR REPORT==== 29-Apr-2018::15:11:36 ===
** State machine bank_statem terminating
** Last event = {{call,{,#Ref}},
                 {withdraw,3}}
** When server state  = {open,#{balance => 0}}
** Reason for termination = error:function_clause
** Callback mode = state_functions
** Stacktrace =
**  [{bank_statem,open,
                  [{call,{,#Ref}},
                   {withdraw,3},
                   #{balance => 0}],
                  [{file,"/app/_build/test/lib/bank_statem/src/bank_statem.erl"},
                   {line,46}]},
...
[{set,{var,1},{call,bank_statem,withdraw,[3]}}]

Shrinking (0 time(s))
[{set,{var,1},{call,bank_statem,withdraw,[3]}}]

=ERROR REPORT==== 29-Apr-2018::15:11:36 ===
** State machine bank_statem terminating
** Last event = {{call,{,#Ref}},
                 {withdraw,3}}
** When server state  = {open,#{balance => 0}}
** Reason for termination = error:function_clause
** Callback mode = state_functions
** Stacktrace =
**  [{bank_statem,open,
                  [{call,{,#Ref}},
                   {withdraw,3},
                   #{balance => 0}],
                  [{file,"/app/_build/test/lib/bank_statem/src/bank_statem.erl"},
                   {line,46}]},
....
===> 
0/1 properties passed, 1 failed
===> Failed test cases:
  prop_bank_statem:prop_test() -> false

We can see that the error is caused by trying to withdraw funds that don’t exist. This can be avoided by adding a pre-condition, to ensure that invalid commands aren’t generated:

precondition(_From, _To, #{balance:=Balance}, {call, bank_statem, withdraw, [Amount]}) ->
    Balance - Amount >= 0;

And also update our model when a withdrawal occurs:

next_state_data(_From, _To, #{balance:=Balance}=Data, _Res, {call, bank_statem, withdraw, [Amount]}) ->
    NewBalance = Balance - Amount,
    Data#{balance:=NewBalance};

At this point, I would expect the property to pass, but it’s still failing:

===> Testing prop_bank_statem:prop_test()
...................!
Failed: After 20 test(s).
A linked process died with reason {function_clause,[{bank_statem,open,[{call,{,#Ref}},{withdraw,9},#{balance=>9}],[{file,[47,97,112,112,47,95,98,117,105,108,100,47,116,101,115,116,47,108,105,98,47,98,97,110,107,95,115,116,97,116,101,109,47,115,114,99,47,98,97,110,107,95,115,116,97,116,101,109,46,101,114,108]},{line,46}]},{gen_statem,call_state_function,5,[{file,[103,101,110,95,115,116,97,116,101,109,46,101,114,108]},{line,1633}]},{gen_statem,loop_event_state_function,6,[{file,[103,101,110,95,115,116,97,116,101,109,46,101,114,108]},{line,1023}]},{proc_lib,init_p_do_apply,3,[{file,[112,114,111,99,95,108,105,98,46,101,114,108]},{line,247}]}]}.

=ERROR REPORT==== 29-Apr-2018::15:14:34 ===
** State machine bank_statem terminating
** Last event = {{call,{,#Ref}},
                 {withdraw,9}}
** When server state  = {open,#{balance => 9}}
** Reason for termination = error:function_clause
** Callback mode = state_functions
** Stacktrace =
**  [{bank_statem,open,
                  [{call,{,#Ref}},
                   {withdraw,9},
                   #{balance => 9}],
                  [{file,"/app/_build/test/lib/bank_statem/src/bank_statem.erl"},
                   {line,46}]},
...

Shrinking .
=ERROR REPORT==== 29-Apr-2018::15:14:34 ===
** State machine bank_statem terminating
** Last event = {{call,{,#Ref}},
                 {withdraw,9}}
** When server state  = {open,#{balance => 9}}
** Reason for termination = error:function_clause
** Callback mode = state_functions
** Stacktrace =
**  [{bank_statem,open,
                  [{call,{,#Ref}},
                   {withdraw,9},
                   #{balance => 9}],
                  [{file,"/app/_build/test/lib/bank_statem/src/bank_statem.erl"},
                   {line,46}]},
...
.
=ERROR REPORT==== 29-Apr-2018::15:14:34 ===
** State machine bank_statem terminating
** Last event = {{call,{,#Ref}},
                 {withdraw,9}}
** When server state  = {open,#{balance => 9}}
** Reason for termination = error:function_clause
** Callback mode = state_functions
** Stacktrace =
**  [{bank_statem,open,
                  [{call,{,#Ref}},
                   {withdraw,9},
                   #{balance => 9}],
                  [{file,"/app/_build/test/lib/bank_statem/src/bank_statem.erl"},
                   {line,46}]},
...
(2 time(s))

=ERROR REPORT==== 29-Apr-2018::15:14:34 ===
** State machine bank_statem terminating
** Last event = {{call,{,#Ref}},
                 {withdraw,9}}
** When server state  = {open,#{balance => 9}}
** Reason for termination = error:function_clause
** Callback mode = state_functions
** Stacktrace =
**  [{bank_statem,open,
                  [{call,{,#Ref}},
                   {withdraw,9},
                   #{balance => 9}],
                  [{file,"/app/_build/test/lib/bank_statem/src/bank_statem.erl"},
                   {line,46}]},
...
[{set,{var,1},{call,bank_statem,deposit,[2]}},{set,{var,2},{call,bank_statem,deposit,[7]}},{set,{var,3},{call,bank_statem,withdraw,[9]}}]
===> 
0/1 properties passed, 1 failed
===> Failed test cases:
  prop_bank_statem:prop_test() -> false

Rather embarrassingly, that’s an actual bug in my code, the guard is checking that the remaining balance is strictly greater than 0. Hurray for property testing!

State machine properties

One of the more interesting corners of the Erlang ecosystem is property based testing. Building on Haskell’s QuickCheck, it allows generation of test cases, similar to “fuzzing”.

The integration with gen_fsm/statem makes stateful testing remarkably easy, by describing the possible transitions for our state machine:

-module(prop_bank_statem).

-include_lib("proper/include/proper.hrl").

-compile(export_all).

prop_test() ->
    ?FORALL(Cmds, proper_fsm:commands(?MODULE),
        begin
            bank_statem:start_link(),
            {History,State,Result} = proper_fsm:run_commands(?MODULE, Cmds),
            bank_statem:stop(),
            ?WHENFAIL(
                io:format("History: ~p\nState: ~p\nResult: ~p\n", [History,State,Result]),
                aggregate(zip(proper_fsm:state_names(History), command_names(Cmds)), Result =:= ok)
            )
        end).

initial_state() -> open.

initial_state_data() -> #{}.

open(_Data) -> [{open, {call, bank_statem, deposit, [pos_integer()]}}].

weight(_FromState, _ToState, _Call) -> 1.

precondition(_From, _To, #{}, {call, _Mod, _Fun, _Args}) -> true.

postcondition(_From, _To, _Data, {call, _Mod, _Fun, _Args}, _Res) -> true.

next_state_data(_From, _To, Data, _Res, {call, _Mod, _Fun, _Args}) ->
    NewData = Data,
    NewData.

To begin with, we’re just checking that a deposit can be made to an open account. This property succeeds (after the default 100 cases):

===> Testing prop_bank_statem:prop_test()
....................................................................................................
OK: Passed 100 test(s).

100% {open,{bank_statem,deposit,1}}
===> 
1/1 properties passed

But, if we switch to using a less strict generator:

open(_Data) -> [{open, {call, bank_statem, deposit, [integer()]}}].

We very quickly hit a failure:

===> Testing prop_bank_statem:prop_test()

=ERROR REPORT==== 29-Apr-2018::14:05:12 ===
** State machine bank_statem terminating
** Last event = {{call,{,#Ref}},
                 {deposit,0}}
** When server state  = {open,#{balance => 0}}
** Reason for termination = error:function_clause
** Callback mode = state_functions
** Stacktrace =
**  [{bank_statem,handle_deposit,
                  [0,
                   #{balance => 0},
                   {,#Ref}],
                  [{file,"/app/_build/test/lib/bank_statem/src/bank_statem.erl"},
                   {line,77}]},

caused by an attempt to make a deposit of 0.

To make things more interesting, we can amend our deposit transition to return the updated balance, and track that in the model:

initial_state_data() -> #{balance=>0}.

...

postcondition(_From, _To, #{balance:=PrevBalance}, {call, bank_statem, deposit, [Amount]}, {deposit_made, UpdatedBalance}) ->
    UpdatedBalance =:= (PrevBalance + Amount);

...

next_state_data(_From, _To, #{balance:=Balance}=Data, _Res, {call, bank_statem, deposit, [Amount]}) ->
    NewBalance = Balance + Amount,
    Data#{balance:=NewBalance};

For this toy example, the model is almost exactly the same as the implementation. And, in fact, the hardest thing with this style of property testing is keeping the model simple, when testing a more complex example.

Next time, we will add the rest of the transitions to our model.

Hold pls

On hold

The next step for our bank account is to place a hold on an account.

account-events-withself-held

Using state functions, this is as simple as:

open({call, From}, place_hold, Data) ->
    {next_state, held, Data, [{reply, From, hold_placed}]};

...

open({call, From}, {deposit, Amount}, Data) ->
    handle_deposit(Amount, Data, From);

...

held({call, From}, {deposit, Amount}, Data) ->
    handle_deposit(Amount, Data, From);

held({call, From}, remove_hold, Data) ->
    {next_state, open, Data, [{reply, From, hold_removed}]}.

handle_deposit(Amount, #{balance:=Balance} = Data, From) when is_number(Amount) andalso Amount > 0 ->
    NewBalance = Balance + Amount,
    {keep_state, Data#{balance:=NewBalance}, [{reply, From, deposit_made}]}.

Now any attempt to withdraw funds from a held account, will cause the gen_statem process to crash (hopefully having previously persisted any important data!).

And using handle event:

handle_event({call, From}, place_hold, open, Data) ->
    {next_state, held, Data, [{reply, From, hold_placed}]};

...

handle_event({call, From}, remove_hold, held, Data) ->
    {next_state, open, Data, [{reply, From, hold_removed}]};

...

handle_event({call, From}, {deposit, Amount}, open, Data) ->
    handle_deposit(Amount, Data, From);

handle_event({call, From}, {deposit, Amount}, held, Data) ->
    handle_deposit(Amount, Data, From);

...

handle_deposit(Amount, #{balance:=Balance} = Data, From) when is_number(Amount) andalso Amount > 0 ->
    NewBalance = Balance + Amount,
    {keep_state, Data#{balance:=NewBalance}, [{reply, From, deposit_made}]}.

Insufficient funds?

We can also add an availableToWithdraw method, with different behaviour for held accounts, without too much hassle:

open({call, From}, available_to_withdraw, #{balance:=Balance} = Data) ->
    {keep_state, Data, [{reply, From, Balance}]};

...

held({call, From}, available_to_withdraw, Data) ->
    {keep_state, Data, [{reply, From, 0}]};

Or:

handle_event({call, From}, available_to_withdraw, open, #{balance:=Balance} = Data) ->
    {keep_state, Data, [{reply, From, Balance}]};

...

handle_event({call, From}, available_to_withdraw, held, Data) ->
    {keep_state, Data, [{reply, From, 0}]};

Handle event function

The main difference between gen_statem and the older gen_fsm, is the addition of a new “handle event function” callback mode, with less restrictions on the state data type.

Converting an existing state machine, is pretty easy. Change the callback mode, and move the existing state functions into one callback:

callback_mode() -> handle_event_function.

handle_event({call, From}, get_balance, open, #{balance:=Balance} = Data) ->
    {keep_state, Data, [{reply, From, Balance}]};

handle_event({call, From}, close, open, Data) ->
    {next_state, closed, Data, [{reply, From, closed}]};

handle_event({call, From}, {deposit, Amount}, open, #{balance:=Balance} = Data) when is_number(Amount) andalso Amount > 0 ->
    NewBalance = Balance + Amount,
    {keep_state, Data#{balance:=NewBalance}, [{reply, From, deposit_made}]};

handle_event({call, From}, {withdraw, Amount}, open, #{balance:=Balance} = Data) when is_number(Amount) andalso (Balance - Amount > 0) ->
    NewBalance = Balance - Amount,
    {keep_state, Data#{balance:=NewBalance}, [{reply, From, withdrawal_made}]};

handle_event({call, From}, reopen, closed, Data) ->
    {next_state, open, Data, [{reply, From, open}]}.

Whether you prefer this style, or the previous version, seems like a matter of personal taste, if you don’t need the extra flexibility.

❤️ the State Machine

Having recently read a post about using state machines in JS, I decided to try implementing the same logic using the (relatively) new gen_statem.

As ever, the code is available here.

The initial example is a naive representation of a bank account:

account-events-withself

The first step is to fill out the gen_statem boilerplate:

-module(bank_statem).

-behaviour(gen_statem).

-export([init/1, callback_mode/0]).
-export([open/3]).

init([]) ->
    {ok, open, #{balance=>0}}.

callback_mode() -> state_functions.

open({call, From}, get_balance, #{balance:=Balance} = Data) ->
    {keep_state, Data, [{reply, From, Balance}]}.

We return an initial state of open, and a map containing the initial balance of 0.

I also added a get_balance call, so we can inspect the current data (which would normally be referred to as the “state” in a gen_server, confusingly).

The first state transition is to close an open account:

open({call, From}, close, Data) ->
    {next_state, closed, Data, [{reply, From, closed}]};

The function name tells you which state this can be called from (open), and the return value tells you that the state machine would transition to a new state (closed), as well as returning a value (the atom closed) to the caller.

Re-opening an account is pretty similar:

closed({call, From}, reopen, Data) ->
    {next_state, open, Data, [{reply, From, open}]}.

As close is only defined for the open state, and reopen is only defined for the closed state, any inappropriate calls will cause the process to crash.

Deposit & withdraw are a little different:

open({call, From}, {deposit, Amount}, #{balance:=Balance} = Data) when is_number(Amount) andalso Amount > 0 ->
    NewBalance = Balance + Amount,
    {keep_state, Data#{balance:=NewBalance}, [{reply, From, deposit_made}]};

open({call, From}, {withdraw, Amount}, #{balance:=Balance} = Data) when is_number(Amount) andalso (Balance - Amount > 0) ->
    NewBalance = Balance - Amount,
    {keep_state, Data#{balance:=NewBalance}, [{reply, From, withdrawal_made}]}.

In that they retain the current state, but the data (in this case, the balance) is updated. Guards have also been used to validate the arguments; again, any other calls will cause an error.

This will seem pretty familiar, if you’ve ever used gen_fsm. Next time, we’ll look at the new alternative callback mode: handle_event_function.

Using rebar3 with Docker

I prefer to avoid installing dev tools on my laptop. I used to create a separate Vagrant instance for each project, but using Docker should provide a lighter weight alternative.

Assuming you already have the Docker tooling installed, the first thing to do is download the base container:

docker pull erlang

Next, download the latest rebar3 binary:

curl -OL https://s3.amazonaws.com/rebar3/rebar3 && chmod +x ./rebar3

At this point, you’re ready to create the app skeleton:

docker run --name $APP_NAME -it --rm -v ${PWD}:/app -w /app erlang ./rebar3 new app $APP_NAME

Annoyingly, rebar assumes that it should create the folder for the project, so you’ll need to move all the generated files up a level (or put up with an extra subdir). Also, the generated files are owned by root, so you probably want to chown the entire dir.

If you’re going to use any rebar plugins, you probably want to put it’s cache dir somewhere that will survive container restarts. Add this to your rebar.config:

{global_rebar_dir, "/app/.cache"}.

And then you can compile your app:

docker run --name $APP_NAME -it --rm -v ${PWD}:/app -w /app -e REBAR_CACHE_DIR=/app erlang ./rebar3 compile

Tennis Kata in Erlang

I’ve recently been reading ploeh’s series on doing the Tennis Kata in F#. I decided to have a go in Erlang, we might not have a fancy type system but we do have pattern matching and partial functions.

You can find the source code to follow along here.

I decided to implement a “game” as a gen_server (when you have a hammer…), I refer you to a previous post for the pre-amble.

Love all

The first test is to check the initial score:

initial_score(Pid) ->
    fun() ->
        ?assertEqual(#{p1=>love,p2=>love}, gen_server:call(Pid, get_score))
    end.

I decided to return a map, rather than a tuple or record, and to use the atom “love” rather than 0. The implementation is pretty trivial:

init([]) ->
    {ok, #{p1=>love,p2=>love}}.

handle_call(get_score, _From, State) ->
    {reply, State, State};

If there was more information in the state map, you might want to return a projection rather than the whole object.

Fifteen love

fifteen_love(Pid) ->
    fun() ->
        ?assertEqual(#{p1=>15,p2=>love}, gen_server:call(Pid, {won_point, p1}))
    end.

This time we need to calculate the new score when a player has won a point:

handle_call({won_point, p1}, _From, State = #{p1:=CurrentScore}) ->
    NewState = State#{p1:=new_score(CurrentScore)},
    {reply, NewState, NewState};

...

new_score(love) -> 15.

Love fifteen

love_fifteen(Pid) ->
    fun() ->
        ?assertEqual(#{p1=>love,p2=>15}, gen_server:call(Pid, {won_point, p2}))
    end.

We can re-use the new_score method from before:

handle_call({won_point, p2}, _From, State = #{p2:=CurrentScore}) ->
    NewState = State#{p2:=new_score(CurrentScore)},
    {reply, NewState, NewState};

Fifteen all

fifteen_all(Pid) ->
    fun() ->
        won_point(Pid, p1),
        ?assertEqual(#{p1=>15,p2=>15}, won_point(Pid, p2))
    end.

won_point(Pid, Player) ->
    gen_server:call(Pid, {won_point, Player}).

No need for any new code, but it seemed like time for a little test refactoring.

Thirty fifteen

thirty_fifteen(Pid) ->
    fun() ->
        won_point(Pid, p1),
        won_point(Pid, p2),
        ?assertEqual(#{p1=>30,p2=>15}, won_point(Pid, p1))
    end.

Now we need to add another function head, to handle the new score:

new_score(love) -> 15;
new_score(15) -> 30.

Thirty all

thirty_all(Pid) ->
    fun() ->
        won_point(Pid, p1),
        won_point(Pid, p2),
        won_point(Pid, p2),
        ?assertEqual(#{p1=>30,p2=>30}, won_point(Pid, p1))
    end.

No new code required.

Forty thirty

forty_thirty(Pid) ->
    fun() ->
        won_point(Pid, p1),
        won_point(Pid, p2),
        won_point(Pid, p1),
        won_point(Pid, p2),
        ?assertEqual(#{p1=>40,p2=>30}, won_point(Pid, p1))
    end.

Again, we need to handle a new score:

...
new_score(30) -> 40.

P1 win

p1_wins(Pid) ->
    fun() ->
        won_point(Pid, p1),
        won_point(Pid, p1),
        won_point(Pid, p1),
        ?assertEqual({game_over, p1}, won_point(Pid, p1))
    end.

This requires a special case, for when the player has 40 points:

handle_call({won_point, p1}, _From, State = #{p1:=40}) ->
    {stop, normal, {game_over, p1}, State};

But now we have a problem, as I skipped over the troublesome “deuce”.

Deuce

deuce(Pid) ->
    fun() ->
        won_point(Pid, p1),
        won_point(Pid, p2),
        won_point(Pid, p1),
        won_point(Pid, p1),
        won_point(Pid, p2),
        ?assertEqual(#{p1=>deuce,p2=>deuce}, won_point(Pid, p2))
    end.

Again, I decided to use an atom, rather than 40-40. But now we need to take both player’s scores into account when determining the new score:

handle_call({won_point, p1}, _From, State = #{p1:=P1Score,p2:=P2Score}) ->
    {NewP1Score, NewP2Score} = new_score(P1Score, P2Score),
    NewState = State#{p1:=NewP1Score, p2:=NewP2Score},
    {reply, NewState, NewState};

...

new_score(love, Score) -> {15, Score};
new_score(15, Score) -> {30, Score};
new_score(30, 40) -> {deuce, deuce};
new_score(30, Score) -> {40, Score}.

Advantage

advantage_p1(Pid) ->
    fun() ->
        won_point(Pid, p1),
        won_point(Pid, p2),
        won_point(Pid, p1),
        won_point(Pid, p1),
        won_point(Pid, p2),
        won_point(Pid, p2),
        ?assertEqual(#{p1=>advantage,p2=>40}, won_point(Pid, p1))
    end.

Another case to handle:

...
new_score(deuce, deuce) -> {advantage, 40}.

Back to Deuce

advantage_p1_back_to_deuce(Pid) ->
    fun() ->
        won_point(Pid, p1),
        won_point(Pid, p2),
        won_point(Pid, p1),
        won_point(Pid, p1),
        won_point(Pid, p2),
        won_point(Pid, p2),
        won_point(Pid, p1),
        ?assertEqual(#{p1=>deuce,p2=>deuce}, won_point(Pid, p2))
    end.

Now we need to add a guard to the win case:

handle_call({won_point, p2}, _From, State = #{p1:=P1Score, p2:=40}) when P1Score =/= advantage ->
    {stop, normal, {game_over, p2}, State};

...

new_score(40, advantage) -> {deuce, deuce}.

Advantage, then win

advantage_p1_then_win(Pid) ->
    fun() ->
        won_point(Pid, p1),
        won_point(Pid, p2),
        won_point(Pid, p1),
        won_point(Pid, p1),
        won_point(Pid, p2),
        won_point(Pid, p2),
        won_point(Pid, p1),
        ?assertEqual({game_over, p1}, won_point(Pid, p1))
    end.

Finally, we need to allow a win from Advantage:

handle_call({won_point, p1}, _From, State = #{p1:=advantage}) ->
    {stop, normal, {game_over, p1}, State};

I think that’s everything covered. As Mark says, “it’s easy enough that it’s fun to play with, but difficult enough that it’s fun to play with”. I personally feel that the use of pattern matching made the intent of the code clearer, but of course YMMV.