Points of intersection of vector with cone

Solution 1:

I'm not handling all the cases where the ray intersects the cone, such as if the ray lies on the cone or if the ray is tangent to the cone, because it's not necessary in my case, but here's the solution I ended up with:

std::array<glm::vec3,2> getLine2ConeIntersection(const glm::vec3 &ap_,const glm::vec3 &ad_ , const glm::vec3 &coneBaseCntr_,const glm::vec3 &coneVertex_,float coneRadius_) const
{
    std::array<glm::vec3,2> pois;
    glm::vec3 axis = (coneBaseCntr_-coneVertex_);
    glm::vec3 theta = (axis/glm::length(axis));
    float m = pow(coneRadius_,2)/pow(glm::length(axis),2);
    glm::vec3 w = (ap_-coneVertex_);

    float a = glm::dot(ad_,ad_) - m*(pow(glm::dot(ad_,theta),2)) - pow(glm::dot(ad_,theta),2);
    float b = 2.f*( glm::dot(ad_,w) - m*glm::dot(ad_,theta)*glm::dot(w,theta) - glm::dot(ad_,theta)*glm::dot(w,theta) );
    float c = glm::dot(w,w) - m*pow(glm::dot(w,theta),2) - pow(glm::dot(w,theta),2);

    float discriminant = pow(b,2) - (4.f*a*c);
    if (discriminant >= 0)
    {
        pois[0] = (ap_+static_cast<float>(((-b) - sqrt(discriminant))/(2.f*a))*ad_);
        pois[1] = (ap_+static_cast<float>(((-b) + sqrt(discriminant))/(2.f*a))*ad_);
    }

    return pois;
}

Where ap_ is a point on the vector and ad_ is it's direction.