How to add a displacement map to a point material in ThreeJS?

I'm new to ThreeJS and trying to create something like asteroids with textures made of particles (something like from this website https://staratlas.com/). Unfortunately, when I tried to apply a displacement map to a points material nothing worked. How can I resolve this issue?

const dispTexture = new THREE.TextureLoader().load('NormalMap.jpg');

const sphere = new THREE.Points(
  new THREE.SphereGeometry(3, 32, 32),
  new THREE.PointsMaterial({
    size: 0.07,
    color: 0xFF325F,
    displacementMap: dispTexture,
    displacementScale : 0.2,
  })
);

scene.add(sphere);

The option, mentioned in comments, with the using of noise. An asteroid of 30K points. Added a simple lighting: the more light on a point, the bigger and brighter it is.

Just in case, codepen: https://codepen.io/prisoner849/pen/mdBzjBy

Maybe it'll be helpful for somebody:

body{
  overflow: hidden;
  margin: 0;
}
<script>
simplexNoise = `
//  Simplex 3D Noise 
//  by Ian McEwan, Ashima Arts
//
vec4 permute(vec4 x){return mod(((x*34.0)+1.0)*x, 289.0);}
vec4 taylorInvSqrt(vec4 r){return 1.79284291400159 - 0.85373472095314 * r;}

float snoise(vec3 v){ 
  const vec2  C = vec2(1.0/6.0, 1.0/3.0) ;
  const vec4  D = vec4(0.0, 0.5, 1.0, 2.0);

// First corner
  vec3 i  = floor(v + dot(v, C.yyy) );
  vec3 x0 =   v - i + dot(i, C.xxx) ;

// Other corners
  vec3 g = step(x0.yzx, x0.xyz);
  vec3 l = 1.0 - g;
  vec3 i1 = min( g.xyz, l.zxy );
  vec3 i2 = max( g.xyz, l.zxy );

  //  x0 = x0 - 0. + 0.0 * C 
  vec3 x1 = x0 - i1 + 1.0 * C.xxx;
  vec3 x2 = x0 - i2 + 2.0 * C.xxx;
  vec3 x3 = x0 - 1. + 3.0 * C.xxx;

// Permutations
  i = mod(i, 289.0 ); 
  vec4 p = permute( permute( permute( 
             i.z + vec4(0.0, i1.z, i2.z, 1.0 ))
           + i.y + vec4(0.0, i1.y, i2.y, 1.0 )) 
           + i.x + vec4(0.0, i1.x, i2.x, 1.0 ));

// Gradients
// ( N*N points uniformly over a square, mapped onto an octahedron.)
  float n_ = 1.0/7.0; // N=7
  vec3  ns = n_ * D.wyz - D.xzx;

  vec4 j = p - 49.0 * floor(p * ns.z *ns.z);  //  mod(p,N*N)

  vec4 x_ = floor(j * ns.z);
  vec4 y_ = floor(j - 7.0 * x_ );    // mod(j,N)

  vec4 x = x_ *ns.x + ns.yyyy;
  vec4 y = y_ *ns.x + ns.yyyy;
  vec4 h = 1.0 - abs(x) - abs(y);

  vec4 b0 = vec4( x.xy, y.xy );
  vec4 b1 = vec4( x.zw, y.zw );

  vec4 s0 = floor(b0)*2.0 + 1.0;
  vec4 s1 = floor(b1)*2.0 + 1.0;
  vec4 sh = -step(h, vec4(0.0));

  vec4 a0 = b0.xzyw + s0.xzyw*sh.xxyy ;
  vec4 a1 = b1.xzyw + s1.xzyw*sh.zzww ;

  vec3 p0 = vec3(a0.xy,h.x);
  vec3 p1 = vec3(a0.zw,h.y);
  vec3 p2 = vec3(a1.xy,h.z);
  vec3 p3 = vec3(a1.zw,h.w);

//Normalise gradients
  vec4 norm = taylorInvSqrt(vec4(dot(p0,p0), dot(p1,p1), dot(p2, p2), dot(p3,p3)));
  p0 *= norm.x;
  p1 *= norm.y;
  p2 *= norm.z;
  p3 *= norm.w;

// Mix final noise value
  vec4 m = max(0.6 - vec4(dot(x0,x0), dot(x1,x1), dot(x2,x2), dot(x3,x3)), 0.0);
  m = m * m;
  return 42.0 * dot( m*m, vec4( dot(p0,x0), dot(p1,x1), 
                                dot(p2,x2), dot(p3,x3) ) );
}
`;
</script>
<script type="module">
import * as THREE from "https://cdn.skypack.dev/[email protected]";
import {OrbitControls} from "https://cdn.skypack.dev/[email protected]/examples/jsm/controls/OrbitControls";

let scene = new THREE.Scene();
let camera = new THREE.PerspectiveCamera(60, innerWidth / innerHeight, 1, 1000);
camera.position.set(0, 0, 10);
let renderer = new THREE.WebGLRenderer({antialias: true});
renderer.setSize(innerWidth, innerHeight);
//renderer.setClearColor(0xffffff)
document.body.appendChild(renderer.domElement);
window.addEventListener("resize", event => {
  camera.aspect = innerWidth / innerHeight;
  camera.updateProjectionMatrix();
  renderer.setSize(innerWidth, innerHeight);
})

let controls = new OrbitControls(camera, renderer.domElement);

let light = new THREE.DirectionalLight(0xffffff, 1);
light.position.set(1, 1, 0);
scene.add(light, new THREE.AmbientLight(0xffffff, 0.5));

//scene.add(new THREE.GridHelper());

let pts = new Array(30000).fill().map(p => {
  return new THREE.Vector3().randomDirection().multiplyScalar(4);
});

let g = new THREE.BufferGeometry().setFromPoints(pts);
let u = {
  time: {value: 0},
  lightPos: {value: new THREE.Vector3()}
}
let m = new THREE.PointsMaterial({
  size: 0.075, 
  color: 0x7fffff,
  //map: new THREE.TextureLoader().load("https://threejs.org/examples/textures/sprites/circle.png"),
  onBeforeCompile: shader => {
    shader.uniforms.lightPos = u.lightPos;
    shader.vertexShader = `
      uniform float time; // just the force of habit to add it :)
      uniform vec3 lightPos;
      varying float vShade;
      
      ${simplexNoise}
      
      float turbulence( vec3 p ) {

        float w = 100.0;
        float t = -.5;

        for (float f = 1.0 ; f <= 10.0 ; f++ ){
          float power = pow( 2.0, f );
          t += snoise( vec3( power * p ) )  / power ;
        }

        return t;

      }
      
      vec3 setFromSphericalCoords( float radius, float phi, float theta ) {
        float sinPhiRadius = sin( phi ) * radius;
        vec3 v = vec3( sinPhiRadius * sin( theta ), cos( phi ) * radius, sinPhiRadius * cos( theta ) );
        return v;
      }
      
      vec2 setFromCartesianCoords( vec3 v ) {
        float radius = sqrt( v.x * v.x + v.y * v.y + v.z * v.z );
        float theta = 0.;
        float phi = 0.;
        if ( radius != 0. ) {
          theta = atan( v.x, v.z );
          phi = acos( clamp( v.y / radius, - 1., 1. ) );
        }
        return vec2(phi, theta);
      }
      
      vec3 getPoint(vec3 p){
        vec3 n = normalize(p);
        float s = turbulence(n * 0.5);
        return p + n * s;
      }
      
      ${shader.vertexShader}
    `.replace(
      `#include <begin_vertex>`,
      `#include <begin_vertex>
        
        vec3 p0 = getPoint(position);
        vec2 spherical = setFromCartesianCoords(position);
        vec2 s = vec2(0.01, 0.);
        vec3 p1 = setFromSphericalCoords(length(position), spherical.x + s.x, spherical.y + s.y);
        vec3 p2 = setFromSphericalCoords(length(position), spherical.x + s.y, spherical.y + s.x);
        p1 = getPoint(p1);
        p2 = getPoint(p2);
        
        vec3 nor = normalize(cross(p1 - p0, p2 - p0));
        
        transformed = p0;
      `
    ).replace(
      `gl_PointSize = size;`,
      `
      vec3 lightDir = normalize(lightPos);
      
      float shade = clamp(dot(nor, lightDir), 0., 1.);
      vShade = shade;
      
      gl_PointSize = size + (shade * size);`
    );
    console.log(shader.vertexShader);
    shader.fragmentShader = `
      varying float vShade;
      ${shader.fragmentShader}
    `.replace(
      `vec4 diffuseColor = vec4( diffuse, opacity );`,
      `
      if(length(gl_PointCoord - 0.5) > 0.5) discard; // make'em round
      float shade = vShade * 0.5 + 0.5;
      vec4 diffuseColor = vec4( diffuse * shade, opacity );`
    );
    console.log(shader.fragmentShader);
  }
});
let p = new THREE.Points(g, m);
scene.add(p);

let clock = new THREE.Clock();

renderer.setAnimationLoop(() => {
  let t = clock.getElapsedTime() * 0.5;
  p.rotation.x = t * 0.271;
  p.rotation.y = t * 0.314;
  p.rotation.z = t * 0.158;
  p.worldToLocal(u.lightPos.value.copy(light.position).add(p.position));
  renderer.render(scene, camera);
})

</script>