I finally got around this problem by handling the authentication outside Firebase with the Google APIs JavaScript client. This solution requires including the Google auth client as documented here. Manually handling the Firebase sign-in flow is documented here.

gapi.auth2.getAuthInstance().signIn()
    .then(function _firebaseSignIn(googleUser) {
      var unsubscribe = firebase.auth().onAuthStateChanged(function(firebaseUser) {
        unsubscribe();
        // Check if we are already signed-in Firebase with the correct user.
        if (!_isUserEqual(googleUser, firebaseUser)) {
          // Build Firebase credential with the Google ID token.
          var credential = firebase.auth.GoogleAuthProvider.credential(
            googleUser.getAuthResponse().id_token);

          // Sign in with credential from the Google user.
          return firebase.auth().signInWithCredential(credential)
            .then(function(result) {
              // other stuff...
            });

The _isUserEqual function:

function _isUserEqual(googleUser, firebaseUser) {
  if (firebaseUser) {
    var providerData = firebaseUser.providerData;
    for (var i = 0; i < providerData.length; i++) {
      if (providerData[i].providerId === firebase.auth.GoogleAuthProvider.PROVIDER_ID &&
        providerData[i].uid === googleUser.getBasicProfile().getId()) {
        // We don't need to reauth the Firebase connection.
        return true;
      }
    }
  }
  return false;
}

Now I can reference the access token like this:

var user = gapi.auth2.getAuthInstance().currentUser.get();
return user.getAuthResponse().access_token;

This still isn't the ideal solution for me, but it works for now, and I'm able to authenticate to both Firebase and the Calendar API.