ggplot, drawing line between points across facets

Using ggplot2, how can I draw a trendline which runs between facets.

library(ggplot2)
df <- data.frame(y=c(1,2,3),x=1,Set=LETTERS[1:3])
ggplot(df,aes(x,y)) + 
  theme_bw() + theme(legend.position=c(0,1),legend.justification=c(0,1)) + 
  geom_point(aes(fill=Set),color="black",shape=21,size=3) + 
  facet_grid(~Set) + 
  xlim(1,5)

Which produces the following:

In the above, I would like to draw a line between the three points, moving across facets.

Updated to ggplot2 V3.0.0

You could do this, but turning clip off might have unwanted consequences,

library(ggplot2)
df <- data.frame(y=c(1,2,3),x=1,Set=LETTERS[1:3])
p <- ggplot(df,aes(x,y)) + 
  theme_bw() + theme(legend.position=c(.01,.99),legend.justification=c(0,1)) + 
  geom_point(aes(fill=Set),color="black",shape=21,size=3) + 
  facet_grid(~Set) + 
  xlim(1,5) 

gb <- ggplot_build(p)
g <- ggplot_gtable(gb)

library(gtable)
library(grid)
# ggplot2 doesn't use native units in data space
# instead, the data is rescaled to npc, i.e from 0 to 1
# so we need to use the build info to convert from data to [0,1]
ranges <- gb$layout$panel_params
data2npc <- function(x, range) scales::rescale(c(range, x), c(0,1))[-c(1,2)]

start <- c(data2npc(1, ranges[[1]][["x.range"]]), 
           data2npc(1, ranges[[1]][["y.range"]]))

end <- c(data2npc(1, ranges[[3]][["x.range"]]), 
           data2npc(3, ranges[[3]][["y.range"]]))
# starting position in the first panel
g <- gtable_add_grob(g, moveToGrob(start[1],start[2]), 
                     t = 8, l = 5)
# draw line to end position in last panel
g <- gtable_add_grob(g, lineToGrob(end[1],end[2]), 
                     t = 8, l = 9, z=Inf)
# turn clip off to see the line across panels
g$layout$clip <- "off"
grid.newpage()
grid.draw(g)