How to draw custom button in Window Titlebar with Windows Forms?

The following will work in XP, I have no Vista machine handy to test it, but I think your issues are steming from an incorrect hWnd somehow. Anyway, on with the poorly commented code.

// The state of our little button
ButtonState _buttState = ButtonState.Normal;
Rectangle _buttPosition = new Rectangle();

[DllImport("user32.dll")]
private static extern IntPtr GetWindowDC(IntPtr hWnd);
[DllImport("user32.dll")]
private static extern int GetWindowRect(IntPtr hWnd, 
                                        ref Rectangle lpRect);
[DllImport("user32.dll")]
private static extern int ReleaseDC(IntPtr hWnd, IntPtr hDC);
protected override void WndProc(ref Message m)
{
    int x, y;
    Rectangle windowRect = new Rectangle();
    GetWindowRect(m.HWnd, ref windowRect);

    switch (m.Msg)
    {
        // WM_NCPAINT
        case 0x85:
        // WM_PAINT
        case 0x0A:
            base.WndProc(ref m);

            DrawButton(m.HWnd);

            m.Result = IntPtr.Zero;

            break;

        // WM_ACTIVATE
        case 0x86:
            base.WndProc(ref m);
            DrawButton(m.HWnd);

            break;

        // WM_NCMOUSEMOVE
        case 0xA0:
            // Extract the least significant 16 bits
            x = ((int)m.LParam << 16) >> 16;
            // Extract the most significant 16 bits
            y = (int)m.LParam >> 16;

            x -= windowRect.Left;
            y -= windowRect.Top;

            base.WndProc(ref m);

            if (!_buttPosition.Contains(new Point(x, y)) && 
                _buttState == ButtonState.Pushed)
            {
                _buttState = ButtonState.Normal;
                DrawButton(m.HWnd);
            }

            break;

        // WM_NCLBUTTONDOWN
        case 0xA1:
            // Extract the least significant 16 bits
            x = ((int)m.LParam << 16) >> 16;
            // Extract the most significant 16 bits
            y = (int)m.LParam >> 16;

            x -= windowRect.Left;
            y -= windowRect.Top;

            if (_buttPosition.Contains(new Point(x, y)))
            {
                _buttState = ButtonState.Pushed;
                DrawButton(m.HWnd);
            }
            else
                base.WndProc(ref m);

            break;

        // WM_NCLBUTTONUP
        case 0xA2:
            // Extract the least significant 16 bits
            x = ((int)m.LParam << 16) >> 16;
            // Extract the most significant 16 bits
            y = (int)m.LParam >> 16;

            x -= windowRect.Left;
            y -= windowRect.Top;

            if (_buttPosition.Contains(new Point(x, y)) &&
                _buttState == ButtonState.Pushed)
            {
                _buttState = ButtonState.Normal;
                // [[TODO]]: Fire a click event for your button 
                //           however you want to do it.
                DrawButton(m.HWnd);
            }
            else
                base.WndProc(ref m);

            break;

        // WM_NCHITTEST
        case 0x84:
            // Extract the least significant 16 bits
            x = ((int)m.LParam << 16) >> 16;
            // Extract the most significant 16 bits
            y = (int)m.LParam >> 16;

            x -= windowRect.Left;
            y -= windowRect.Top;

            if (_buttPosition.Contains(new Point(x, y)))
                m.Result = (IntPtr)18; // HTBORDER
            else
                base.WndProc(ref m);

            break;

        default:
            base.WndProc(ref m);
            break;
    }
}

private void DrawButton(IntPtr hwnd)
{
    IntPtr hDC = GetWindowDC(hwnd);
    int x, y;

    using (Graphics g = Graphics.FromHdc(hDC))
    {
        // Work out size and positioning
        int CaptionHeight = Bounds.Height - ClientRectangle.Height;
        Size ButtonSize = SystemInformation.CaptionButtonSize;
        x = Bounds.Width - 4 * ButtonSize.Width;
        y = (CaptionHeight - ButtonSize.Height) / 2;
        _buttPosition.Location = new Point(x, y);

        // Work out color
        Brush color;
        if (_buttState == ButtonState.Pushed)
            color = Brushes.LightGreen;
        else
            color = Brushes.Red;

        // Draw our "button"
        g.FillRectangle(color, x, y, ButtonSize.Width, ButtonSize.Height);
    }

    ReleaseDC(hwnd, hDC);
}

private void Form1_Load(object sender, EventArgs e)
{
    _buttPosition.Size = SystemInformation.CaptionButtonSize;
}

I know it's been long since the last answer but this really helped me recently and I like to update the code provided by Chris with my comments and modifications. The version runs perfectly on Win XP and Win 2003. On Win 2008 ot has a small bug that I was not able to identify, when resizing windows. Works on Vista too (no-Aero) but note that the title bar buttons are not square and button dimensions should take that into account.

 switch (m.Msg)
            {
                // WM_NCPAINT / WM_PAINT        
                case 0x85:
                case 0x0A:
                    //Call base method
                    base.WndProc(ref m);
                    //we have 3 buttons in the corner of the window. So first's new button left coord is offseted by 4 widths
                    int crt = 4;
                    //navigate trough all titlebar buttons on the form
                    foreach (TitleBarImageButton crtBtn in titleBarButtons.Values)
                    {
                        //Calculate button coordinates
                        p.X = (Bounds.Width - crt * crtBtn.Size.Width);
                        p.Y = (Bounds.Height - ClientRectangle.Height - crtBtn.Size.Height) / 2;
                        //Initialize button and draw
                        crtBtn.Location = p;
                        crtBtn.ButtonState = ImageButtonState.NORMAL;
                        crtBtn.DrawButton(m.HWnd);
                        //increment button left coord location offset
                        crt++;
                    }
                    m.Result = IntPtr.Zero;
                    break;
                // WM_ACTIVATE      
                case 0x86:
                    //Call base method
                    base.WndProc(ref m);
                    //Draw each button
                    foreach (TitleBarImageButton crtBtn in titleBarButtons.Values)
                    {
                        crtBtn.ButtonState = ImageButtonState.NORMAL;
                        crtBtn.DrawButton(m.HWnd);
                    }
                    break;
                // WM_NCMOUSEMOVE        
                case 0xA0:
                    //Get current mouse position
                    p.X = ((int)m.LParam << 16) >> 16;// Extract the least significant 16 bits            
                    p.Y = (int)m.LParam >> 16;        // Extract the most significant 16 bits          
                    p.X -= windowRect.Left;
                    p.Y -= windowRect.Top;

                    //Call base method
                    base.WndProc(ref m);

                    ImageButtonState newButtonState;
                    foreach (TitleBarImageButton crtBtn in titleBarButtons.Values)
                    {
                        if (crtBtn.HitTest(p))
                        {//mouse is over the current button
                            if (crtBtn.MouseButtonState == MouseButtonState.PRESSED)
                                //button is pressed - set pressed state
                                newButtonState = ImageButtonState.PRESSED;
                            else
                                //button not pressed - set hoover state
                                newButtonState = ImageButtonState.HOOVER;
                        }
                        else
                        {
                            //mouse not over the current button - set normal state
                            newButtonState = ImageButtonState.NORMAL;
                        }

                        //if button state not modified, do not repaint it.
                        if (newButtonState != crtBtn.ButtonState)
                        {
                            crtBtn.ButtonState = newButtonState;
                            crtBtn.DrawButton(m.HWnd);
                        }
                    }
                    break;
                // WM_NCLBUTTONDOWN     
                case 0xA1:
                    //Get current mouse position
                    p.X = ((int)m.LParam << 16) >> 16;// Extract the least significant 16 bits
                    p.Y = (int)m.LParam >> 16;        // Extract the most significant 16 bits      
                    p.X -= windowRect.Left;
                    p.Y -= windowRect.Top;

                    //Call base method
                    base.WndProc(ref m);

                    foreach (TitleBarImageButton crtBtn in titleBarButtons.Values)
                    {
                        if (crtBtn.HitTest(p))
                        {
                            crtBtn.MouseButtonState = MouseButtonState.PRESSED;
                            crtBtn.ButtonState = ImageButtonState.PRESSED;
                            crtBtn.DrawButton(m.HWnd);
                        }
                    }
                    break;
                // WM_NCLBUTTONUP   
                case 0xA2:
                case 0x202:
                    //Get current mouse position
                    p.X = ((int)m.LParam << 16) >> 16;// Extract the least significant 16 bits   
                    p.Y = (int)m.LParam >> 16;        // Extract the most significant 16 bits 
                    p.X -= windowRect.Left;
                    p.Y -= windowRect.Top;

                    //Call base method
                    base.WndProc(ref m);
                    foreach (TitleBarImageButton crtBtn in titleBarButtons.Values)
                    {
                        //if button is press
                        if (crtBtn.ButtonState == ImageButtonState.PRESSED)
                        {
                            //Rasie button's click event
                            crtBtn.OnClick(EventArgs.Empty);

                            if (crtBtn.HitTest(p))
                                crtBtn.ButtonState = ImageButtonState.HOOVER;
                            else
                                crtBtn.ButtonState = ImageButtonState.NORMAL;
                        }

                        crtBtn.MouseButtonState = MouseButtonState.NOTPESSED;
                        crtBtn.DrawButton(m.HWnd);
                    }
                    break;
                // WM_NCHITTEST    
                case 0x84:
                    //Get current mouse position
                    p.X = ((int)m.LParam << 16) >> 16;// Extract the least significant 16 bits
                    p.Y = (int)m.LParam >> 16;        // Extract the most significant 16 bits
                    p.X -= windowRect.Left;
                    p.Y -= windowRect.Top;

                    bool isAnyButtonHit = false;
                    foreach (TitleBarImageButton crtBtn in titleBarButtons.Values)
                    {
                        //if mouse is over the button, or mouse is pressed 
                        //(do not process messages when mouse was pressed on a button)
                        if (crtBtn.HitTest(p) || crtBtn.MouseButtonState == MouseButtonState.PRESSED)
                        {
                            //return 18 (do not process further)
                            m.Result = (IntPtr)18;
                            //we have a hit
                            isAnyButtonHit = true;
                            //return 
                            break;
                        }
                        else
                        {//mouse is not pressed and not over the button, redraw button if needed  
                            if (crtBtn.ButtonState != ImageButtonState.NORMAL)
                            {
                                crtBtn.ButtonState = ImageButtonState.NORMAL;
                                crtBtn.DrawButton(m.HWnd);
                            }
                        }
                    }
                    //if we have a hit, do not process further
                    if (!isAnyButtonHit)
                        //Call base method
                        base.WndProc(ref m);
                    break;
                default:
                    //Call base method
                    base.WndProc(ref m);
                    //Console.WriteLine(m.Msg + "(0x" + m.Msg.ToString("x") + ")");
                    break;
            }

The code demonstrates the messages that heve to be treated and how to treat them. The code uses a collection of custom TitleBarButton objets. That class is too big to be included here but I can provide it if needed along with an example.


Drawing seems to be the easy part, the following will do that:

[Edit: Code removed, see my other answer]

The real problem is changing the state and detecting clicks on the button... for that you'll need to hook into the global message handler for the program, .NET seems to hide the mouse events for a form while not in the actual container areas (ie. mouse moves and clicks on the title bar). I'm looking for info on that, found it now, I'm working on it, shouldn't be too hard... If we can figure out what these messages are actually passing.