How to place grobs with annotation_custom() at precise areas of the plot region?

Here's how I would approach this,

library(gtable)
library(ggplot2)
library(plyr)

set.seed(1)
d <- data.frame(x=rep(1:10, 5),
                y=rnorm(50),
                g = gl(5,10))

# example plot
p <- ggplot(d, aes(x,y,colour=g)) +
  geom_line() +
  scale_x_continuous(expand=c(0,0))+
  theme(legend.position="top",
        plot.margin=unit(c(1,0,0,0),"line"))

# dummy data for the legend plot
# built with the same y axis (same limits, same expand factor)
d2 <- ddply(d, "g", summarise, x=0, y=y[length(y)])
d2$lab <- paste0("line #", seq_len(nrow(d2)))

plegend <- ggplot(d, aes(x,y, colour=g)) +
  geom_blank() +
  geom_segment(data=d2, aes(x=2, xend=0, y=y, yend=y), 
               arrow=arrow(length=unit(2,"mm"), type="closed")) +
  geom_text(data=d2, aes(x=2.5,label=lab), hjust=0) +
  scale_x_continuous(expand=c(0,0)) +
  guides(colour="none")+
  theme_minimal() + theme(line=element_blank(),
                          text=element_blank(),
                          panel.background=element_rect(fill="grey95", linetype=2))

# extract the panel only, we don't need the rest
gl <- gtable_filter(ggplotGrob(plegend), "panel")

# add a cell next to the main plot panel, and insert gl there
g <- ggplotGrob(p)
index <- subset(g$layout, name == "panel")
g <- gtable_add_cols(g, unit(1, "strwidth", "line # 1") + unit(1, "cm"))
g <- gtable_add_grob(g, gl, t = index$t, l=ncol(g), 
                     b=index$b, r=ncol(g))
grid.newpage()
grid.draw(g)

It should be straight-forward to adapt the "legend" plot with specific tags and locations (left as an exercise for the interested reader).

Maybe this can illustrate annotation_custom,

myGrob <- grobTree(rectGrob(gp=gpar(fill="red", alpha=0.5)),
                   segmentsGrob(x0=0, x1=1, y0=0, y1=1, default.units="npc"))

myGrob2 <- grobTree(rectGrob(gp=gpar(fill="blue", alpha=0.5)),
                   segmentsGrob(x0=0, x1=1, y0=0, y1=1, default.units="npc"))

p <- qplot(1:10, 1:10) + theme(plot.margin=unit(c(0, 3, 0, 0), "cm")) +
  annotation_custom(myGrob, xmin=5, xmax=6, ymin=3.5, ymax=5.5) +
  annotate("segment", x=5, xend=6, y=3, yend=5, colour="red") +
  annotation_custom(myGrob2, xmin=8, xmax=12, ymin=3.5, ymax=5.5) 

p

g <- ggplotGrob(p)
g$layout$clip[g$layout$name=="panel"] <- "off"
grid.draw(g)

There's a weird bug apparently, whereby if I reuse myGrob instead of myGrob2, it ignores the placement coordinates the second time and stacks it up with the first layer. This function is really buggy.