After checking out that library, here's what I've got.

return QueryBuilder::for(User::class)
    ->with('phoneNumbers') // <-- I think you can use `allowedIncludes` here.
    ->allowedFilters([ // <-- I believe that `withTrashed` is a scope query,
                       // so you can use this. You cannot use `allowedIncludes`
                       // because it works with relations.
        Filter::scope('withTrashed')
    ])
    ->get();