How to create custom 3d model in JavaFX 8?

nothing appeared in the scene

Your sample mesh rendered OK for me.

Perhaps you didn't have your camera setup correctly or have your mesh scaled so that it was visible.

Your sample mesh doesn't do much, it is one triangle facing towards the camera and a second triangle facing away from the camera.

And is it possible to create them without texture(I want wireframe model)?

Yes, set the DrawMode for your mesh view to Line.

Sample Program Explanation

I changed the order of your faces so that they both face the same direction so that you get a square facing the viewer rather than one triangle facing towards the viewer and one away from the viewer:

int[] faces = {
    2, 2, 1, 1, 0, 0,
    2, 2, 3, 3, 1, 1
};

Also the texture map needs to change (for the above face array to get the right orientation on the texture so it is not upside down).

float[] texCoords = {
    1, 1,
    1, 0,
    0, 1,
    0, 0
};

I set up a cull back control and animated rotating the model so that you could see the "back" sides of the triangles (in black) and it is clear what is being rendered. I also added the ability to toggle a diffuse map (some marble) for a texture or a wireframe mode.

Sample Program Output

Sample Program

import javafx.animation.*;
import javafx.application.Application;
import javafx.beans.binding.Bindings;
import javafx.geometry.Insets;
import javafx.scene.*;
import javafx.scene.control.CheckBox;
import javafx.scene.image.Image;
import javafx.scene.layout.VBox;
import javafx.scene.paint.*;
import javafx.scene.shape.*;
import javafx.scene.transform.Rotate;
import javafx.stage.Stage;
import javafx.util.Duration;

public class InlineModelViewer extends Application {

  private static final int VIEWPORT_SIZE = 800;

  private static final double MODEL_SCALE_FACTOR = 40;
  private static final double MODEL_X_OFFSET = 0;
  private static final double MODEL_Y_OFFSET = 0;
  private static final double MODEL_Z_OFFSET = VIEWPORT_SIZE / 2;

  private static final String textureLoc = "https://www.sketchuptextureclub.com/public/texture_f/slab-marble-emperador-cream-light-preview.jpg";

  private Image texture;
  private PhongMaterial texturedMaterial = new PhongMaterial();

  private MeshView meshView = loadMeshView();

  private MeshView loadMeshView() {
    float[] points = {
        -5, 5, 0,
        -5, -5, 0,
        5, 5, 0,
        5, -5, 0
    };
    float[] texCoords = {
        1, 1,
        1, 0,
        0, 1,
        0, 0
    };
    int[] faces = {
        2, 2, 1, 1, 0, 0,
        2, 2, 3, 3, 1, 1
    };

    TriangleMesh mesh = new TriangleMesh();
    mesh.getPoints().setAll(points);
    mesh.getTexCoords().setAll(texCoords);
    mesh.getFaces().setAll(faces);

    return new MeshView(mesh);
  }

  private Group buildScene() {
    meshView.setTranslateX(VIEWPORT_SIZE / 2 + MODEL_X_OFFSET);
    meshView.setTranslateY(VIEWPORT_SIZE / 2 * 9.0 / 16 + MODEL_Y_OFFSET);
    meshView.setTranslateZ(VIEWPORT_SIZE / 2 + MODEL_Z_OFFSET);
    meshView.setScaleX(MODEL_SCALE_FACTOR);
    meshView.setScaleY(MODEL_SCALE_FACTOR);
    meshView.setScaleZ(MODEL_SCALE_FACTOR);

    return new Group(meshView);
  }

  @Override
  public void start(Stage stage) {
    texture = new Image(textureLoc);
    texturedMaterial.setDiffuseMap(texture);

    Group group = buildScene();

    RotateTransition rotate = rotate3dGroup(group);

    VBox layout = new VBox(
        createControls(rotate),
        createScene3D(group)
    );

    stage.setTitle("Model Viewer");

    Scene scene = new Scene(layout, Color.CORNSILK);
    stage.setScene(scene);
    stage.show();
  }

  private SubScene createScene3D(Group group) {
    SubScene scene3d = new SubScene(group, VIEWPORT_SIZE, VIEWPORT_SIZE * 9.0/16, true, SceneAntialiasing.BALANCED);
    scene3d.setFill(Color.rgb(10, 10, 40));
    scene3d.setCamera(new PerspectiveCamera());
    return scene3d;
  }

  private VBox createControls(RotateTransition rotateTransition) {
    CheckBox cull      = new CheckBox("Cull Back");
    meshView.cullFaceProperty().bind(
        Bindings.when(
            cull.selectedProperty())
              .then(CullFace.BACK)
              .otherwise(CullFace.NONE)
    );
    CheckBox wireframe = new CheckBox("Wireframe");
    meshView.drawModeProperty().bind(
        Bindings.when(
            wireframe.selectedProperty())
              .then(DrawMode.LINE)
              .otherwise(DrawMode.FILL)
    );

    CheckBox rotate = new CheckBox("Rotate");
    rotate.selectedProperty().addListener(observable -> {
      if (rotate.isSelected()) {
        rotateTransition.play();
      } else {
        rotateTransition.pause();
      }
    });

    CheckBox texture = new CheckBox("Texture");
    meshView.materialProperty().bind(
        Bindings.when(
            texture.selectedProperty())
              .then(texturedMaterial)
              .otherwise((PhongMaterial) null)
    );

    VBox controls = new VBox(10, rotate, texture, cull, wireframe);
    controls.setPadding(new Insets(10));
    return controls;
  }

  private RotateTransition rotate3dGroup(Group group) {
    RotateTransition rotate = new RotateTransition(Duration.seconds(10), group);
    rotate.setAxis(Rotate.Y_AXIS);
    rotate.setFromAngle(0);
    rotate.setToAngle(360);
    rotate.setInterpolator(Interpolator.LINEAR);
    rotate.setCycleCount(RotateTransition.INDEFINITE);

    return rotate;
  }

  public static void main(String[] args) {
    System.setProperty("prism.dirtyopts", "false");
    launch(args);
  }
}

Can anybody explain how to create my own 3d models

That is too broad a question for StackOverflow. There are universities and art colleges that hand out diplomas in that kind of stuff.

Can anyone explain why Mesh.setAll takes float[] while the rest of API uses double?

The JavaFX 3D implementation provides a wrapper around native APIs that communicate with graphics hardware (e.g. DirectX and OpenGL). These APIs work with float precision rather than double precision. Using float[] directly in the API means that the mesh model data can be stored more efficiently and directly mapped to the underlying graphics APIs than if double[] or an ObservableList<Double> were used.

Step 1:List the points

cube.getPoints().addAll(
    0, 0, 100,      //P0
    100, 0, 100,    //P1
    0, 100, 100,    //P2
    100, 100, 100,  //P3
    0, 0, 0,        //P4
    100, 0, 0,      //P5
    0, 100, 0,      //P6
    100, 100, 0     //P7
);

Step 2:List the texture points

cube.getTexCoords().addAll(
    0.25f, 0,       //T0
    0.5f, 0,        //T1
    0, 0.25f,       //T2
    0.25f, 0.25f,   //T3
    0.5f, 0.25f,    //T4
    0.75f, 0.25f,   //T5
    1, 0.25f,       //T6
    0, 0.5f,        //T7
    0.25f, 0.5f,    //T8
    0.5f, 0.5f,     //T9
    0.75f, 0.5f,    //T10
    1, 0.5f,        //T11
    0.25f, 0.75f,   //T12
    0.5f, 0.75f     //T13
);

Step 3: List the faces (it Mixed 3D points and Texture points counter-clockwise (or right-hand rule))

cube.getFaces().addAll(
    5,1,4,0,0,3     //P5,T1 ,P4,T0  ,P0,T3
    ,5,1,0,3,1,4    //P5,T1 ,P0,T3  ,P1,T4
    ,0,3,4,2,6,7    //P0,T3 ,P4,T2  ,P6,T7
    ,0,3,6,7,2,8    //P0,T3 ,P6,T7  ,P2,T8
    ,1,4,0,3,2,8    //P1,T4 ,P0,T3  ,P2,T8
    ,1,4,2,8,3,9    //P1,T4 ,P2,T8  ,P3,T9
    ,5,5,1,4,3,9    //P5,T5 ,P1,T4  ,P3,T9
    ,5,5,3,9,7,10   //P5,T5 ,P3,T9  ,P7,T10
    ,4,6,5,5,7,10   //P4,T6 ,P5,T5  ,P7,T10
    ,4,6,7,10,6,11  //P4,T6 ,P7,T10 ,P6,T11
    ,3,9,2,8,6,12   //P3,T9 ,P2,T8  ,P6,T12
    ,3,9,6,12,7,13  //P3,T9 ,P6,T12 ,P7,T13
);