How to implement custom authentication in ASP.NET MVC 5

Yes, you can. Authentication and Authorization parts work independently. If you have your own authentication service you can just use OWIN's authorization part. Consider you already have a UserManager which validates username and password. Therefore you can write the following code in your post back login action:

[HttpPost]
public ActionResult Login(string username, string password)
{
    if (new UserManager().IsValid(username, password))
    {
        var ident = new ClaimsIdentity(
          new[] { 
              // adding following 2 claim just for supporting default antiforgery provider
              new Claim(ClaimTypes.NameIdentifier, username),
              new Claim("http://schemas.microsoft.com/accesscontrolservice/2010/07/claims/identityprovider", "ASP.NET Identity", "http://www.w3.org/2001/XMLSchema#string"),

              new Claim(ClaimTypes.Name,username),

              // optionally you could add roles if any
              new Claim(ClaimTypes.Role, "RoleName"),
              new Claim(ClaimTypes.Role, "AnotherRole"),

          },
          DefaultAuthenticationTypes.ApplicationCookie);

        HttpContext.GetOwinContext().Authentication.SignIn(
           new AuthenticationProperties { IsPersistent = false }, ident);
        return RedirectToAction("MyAction"); // auth succeed 
    }
    // invalid username or password
    ModelState.AddModelError("", "invalid username or password");
    return View();
}

And your user manager can be something like this:

class UserManager
{
    public bool IsValid(string username, string password)
    {
         using(var db=new MyDbContext()) // use your DbConext
         {
             // for the sake of simplicity I use plain text passwords
             // in real world hashing and salting techniques must be implemented   
             return db.Users.Any(u=>u.Username==username 
                 && u.Password==password); 
         }
    }
}

In the end, you can protect your actions or controllers by adding an Authorize attribute.

[Authorize]
public ActionResult MySecretAction()
{
    // all authorized users can use this method
    // we have accessed current user principal by calling also
    // HttpContext.User
}

[Authorize(Roles="Admin")]
public ActionResult MySecretAction()
{
    // just Admin users have access to this method
}