Aqui está a resposta:
scale_fab_in.xml
<?xml version="1.0" encoding="utf-8"?>
<scale xmlns:android="http://schemas.android.com/apk/res/android"
android:duration="500"
android:fromXScale="0"
android:fromYScale="0"
android:pivotX="50%"
android:pivotY="50%"
android:toXScale="1"
android:toYScale="1"
android:interpolator="@android:interpolator/overshoot"/>
scale_fab_out.xml
<?xml version="1.0" encoding="utf-8"?>
<scale xmlns:android="http://schemas.android.com/apk/res/android"
android:duration="400"
android:fromXScale="1"
android:fromYScale="1"
android:pivotX="50%"
android:pivotY="50%"
android:toXScale="0"
android:toYScale="0"
android:interpolator="@android:interpolator/overshoot"/>
A partir daqui:
https://stackoverflow.com/questions/27922191/floating-action-button-animation/27922318#27922318