TensorFlow: Blas GEMM launch failed
When I'm trying to use TensorFlow with Keras using the gpu, I'm getting this error message:
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\__main__.py:2: UserWarning: Update your `fit_generator` call to the Keras 2 API: `fit_generator(<keras.pre..., 37800, epochs=2, validation_data=<keras.pre..., validation_steps=4200)`
from ipykernel import kernelapp as app
Epoch 1/2
InternalError Traceback (most recent call last)
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1038 try:
-> 1039 return fn(*args)
1040 except errors.OpError as e:
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
1020 feed_dict, fetch_list, target_list,
-> 1021 status, run_metadata)
1022
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\contextlib.py in __exit__(self, type, value, traceback)
65 try:
---> 66 next(self.gen)
67 except StopIteration:
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\errors_impl.py in raise_exception_on_not_ok_status()
465 compat.as_text(pywrap_tensorflow.TF_Message(status)),
--> 466 pywrap_tensorflow.TF_GetCode(status))
467 finally:
InternalError: Blas GEMM launch failed : a.shape=(64, 784), b.shape=(784, 10), m=64, n=10, k=784
[[Node: dense_1/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](flatten_1/Reshape, dense_1/kernel/read)]]
During handling of the above exception, another exception occurred:
InternalError Traceback (most recent call last)
<ipython-input-13-2a52d1079a66> in <module>()
1 history=model.fit_generator(batches, batches.n, nb_epoch=2,
----> 2 validation_data=val_batches, nb_val_samples=val_batches.n)
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
86 warnings.warn('Update your `' + object_name +
87 '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 88 return func(*args, **kwargs)
89 wrapper._legacy_support_signature = inspect.getargspec(func)
90 return wrapper
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\models.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_q_size, workers, pickle_safe, initial_epoch)
1108 workers=workers,
1109 pickle_safe=pickle_safe,
-> 1110 initial_epoch=initial_epoch)
1111
1112 @interfaces.legacy_generator_methods_support
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
86 warnings.warn('Update your `' + object_name +
87 '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 88 return func(*args, **kwargs)
89 wrapper._legacy_support_signature = inspect.getargspec(func)
90 return wrapper
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\engine\training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_q_size, workers, pickle_safe, initial_epoch)
1888 outs = self.train_on_batch(x, y,
1889 sample_weight=sample_weight,
-> 1890 class_weight=class_weight)
1891
1892 if not isinstance(outs, list):
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\engine\training.py in train_on_batch(self, x, y, sample_weight, class_weight)
1631 ins = x + y + sample_weights
1632 self._make_train_function()
-> 1633 outputs = self.train_function(ins)
1634 if len(outputs) == 1:
1635 return outputs[0]
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\backend\tensorflow_backend.py in __call__(self, inputs)
2227 session = get_session()
2228 updated = session.run(self.outputs + [self.updates_op],
-> 2229 feed_dict=feed_dict)
2230 return updated[:len(self.outputs)]
2231
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in run(self, fetches, feed_dict, options, run_metadata)
776 try:
777 result = self._run(None, fetches, feed_dict, options_ptr,
--> 778 run_metadata_ptr)
779 if run_metadata:
780 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
980 if final_fetches or final_targets:
981 results = self._do_run(handle, final_targets, final_fetches,
--> 982 feed_dict_string, options, run_metadata)
983 else:
984 results = []
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1030 if handle is None:
1031 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
-> 1032 target_list, options, run_metadata)
1033 else:
1034 return self._do_call(_prun_fn, self._session, handle, feed_dict,
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1050 except KeyError:
1051 pass
-> 1052 raise type(e)(node_def, op, message)
1053
1054 def _extend_graph(self):
InternalError: Blas GEMM launch failed : a.shape=(64, 784), b.shape=(784, 10), m=64, n=10, k=784
[[Node: dense_1/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](flatten_1/Reshape, dense_1/kernel/read)]]
Caused by op 'dense_1/MatMul', defined at:
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\__main__.py", line 3, in <module>
app.launch_new_instance()
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\traitlets\config\application.py", line 658, in launch_instance
app.start()
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelapp.py", line 477, in start
ioloop.IOLoop.instance().start()
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tornado\ioloop.py", line 888, in start
handler_func(fd_obj, events)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelbase.py", line 235, in dispatch_shell
handler(stream, idents, msg)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\zmqshell.py", line 533, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\IPython\core\interactiveshell.py", line 2683, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\IPython\core\interactiveshell.py", line 2787, in run_ast_nodes
if self.run_code(code, result):
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\IPython\core\interactiveshell.py", line 2847, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-10-1e7a3b259f23>", line 4, in <module>
model.add(Dense(10, activation='softmax'))
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\models.py", line 466, in add
output_tensor = layer(self.outputs[0])
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\engine\topology.py", line 585, in __call__
output = self.call(inputs, **kwargs)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\layers\core.py", line 840, in call
output = K.dot(inputs, self.kernel)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\backend\tensorflow_backend.py", line 936, in dot
out = tf.matmul(x, y)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\ops\math_ops.py", line 1801, in matmul
a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 1263, in _mat_mul
transpose_b=transpose_b, name=name)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 768, in apply_op
op_def=op_def)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\ops.py", line 1228, in __init__
self._traceback = _extract_stack()
InternalError (see above for traceback): Blas GEMM launch failed : a.shape=(64, 784), b.shape=(784, 10), m=64, n=10, k=784
[[Node: dense_1/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](flatten_1/Reshape, dense_1/kernel/read)]]
When I'm trying to use TensorFlow with Keras using the cpu, I'm getting this error message:
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\__main__.py:5: UserWarning: Update your `fit_generator` call to the Keras 2 API: `fit_generator(<keras.pre..., 37800, validation_steps=4200, validation_data=<keras.pre..., epochs=2)`
Epoch 1/2
---------------------------------------------------------------------------
InternalError Traceback (most recent call last)
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1038 try:
-> 1039 return fn(*args)
1040 except errors.OpError as e:
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
1020 feed_dict, fetch_list, target_list,
-> 1021 status, run_metadata)
1022
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\contextlib.py in __exit__(self, type, value, traceback)
65 try:
---> 66 next(self.gen)
67 except StopIteration:
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\errors_impl.py in raise_exception_on_not_ok_status()
465 compat.as_text(pywrap_tensorflow.TF_Message(status)),
--> 466 pywrap_tensorflow.TF_GetCode(status))
467 finally:
InternalError: Blas GEMM launch failed : a.shape=(64, 784), b.shape=(784, 10), m=64, n=10, k=784
[[Node: dense_1/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](flatten_1/Reshape, dense_1/kernel/read)]]
[[Node: Assign_3/_84 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_374_Assign_3", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
During handling of the above exception, another exception occurred:
InternalError Traceback (most recent call last)
<ipython-input-14-f66b4d3d5b88> in <module>()
3 with tf.device('/cpu:0'):
4 history=model.fit_generator(batches, batches.n, nb_epoch=2,
----> 5 validation_data=val_batches, nb_val_samples=val_batches.n)
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
86 warnings.warn('Update your `' + object_name +
87 '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 88 return func(*args, **kwargs)
89 wrapper._legacy_support_signature = inspect.getargspec(func)
90 return wrapper
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\models.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_q_size, workers, pickle_safe, initial_epoch)
1108 workers=workers,
1109 pickle_safe=pickle_safe,
-> 1110 initial_epoch=initial_epoch)
1111
1112 @interfaces.legacy_generator_methods_support
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
86 warnings.warn('Update your `' + object_name +
87 '` call to the Keras 2 API: ' + signature, stacklevel=2)
---> 88 return func(*args, **kwargs)
89 wrapper._legacy_support_signature = inspect.getargspec(func)
90 return wrapper
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\engine\training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_q_size, workers, pickle_safe, initial_epoch)
1888 outs = self.train_on_batch(x, y,
1889 sample_weight=sample_weight,
-> 1890 class_weight=class_weight)
1891
1892 if not isinstance(outs, list):
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\engine\training.py in train_on_batch(self, x, y, sample_weight, class_weight)
1631 ins = x + y + sample_weights
1632 self._make_train_function()
-> 1633 outputs = self.train_function(ins)
1634 if len(outputs) == 1:
1635 return outputs[0]
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\backend\tensorflow_backend.py in __call__(self, inputs)
2227 session = get_session()
2228 updated = session.run(self.outputs + [self.updates_op],
-> 2229 feed_dict=feed_dict)
2230 return updated[:len(self.outputs)]
2231
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in run(self, fetches, feed_dict, options, run_metadata)
776 try:
777 result = self._run(None, fetches, feed_dict, options_ptr,
--> 778 run_metadata_ptr)
779 if run_metadata:
780 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
980 if final_fetches or final_targets:
981 results = self._do_run(handle, final_targets, final_fetches,
--> 982 feed_dict_string, options, run_metadata)
983 else:
984 results = []
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1030 if handle is None:
1031 return self._do_call(_run_fn, self._session, feed_dict, fetch_list,
-> 1032 target_list, options, run_metadata)
1033 else:
1034 return self._do_call(_prun_fn, self._session, handle, feed_dict,
C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
1050 except KeyError:
1051 pass
-> 1052 raise type(e)(node_def, op, message)
1053
1054 def _extend_graph(self):
InternalError: Blas GEMM launch failed : a.shape=(64, 784), b.shape=(784, 10), m=64, n=10, k=784
[[Node: dense_1/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](flatten_1/Reshape, dense_1/kernel/read)]]
[[Node: Assign_3/_84 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_374_Assign_3", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
Caused by op 'dense_1/MatMul', defined at:
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\__main__.py", line 3, in <module>
app.launch_new_instance()
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\traitlets\config\application.py", line 658, in launch_instance
app.start()
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelapp.py", line 477, in start
ioloop.IOLoop.instance().start()
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\ioloop.py", line 177, in start
super(ZMQIOLoop, self).start()
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tornado\ioloop.py", line 888, in start
handler_func(fd_obj, events)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\zmqstream.py", line 440, in _handle_events
self._handle_recv()
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\zmqstream.py", line 472, in _handle_recv
self._run_callback(callback, msg)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\zmq\eventloop\zmqstream.py", line 414, in _run_callback
callback(*args, **kwargs)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tornado\stack_context.py", line 277, in null_wrapper
return fn(*args, **kwargs)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelbase.py", line 235, in dispatch_shell
handler(stream, idents, msg)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\ipkernel.py", line 196, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\ipykernel\zmqshell.py", line 533, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\IPython\core\interactiveshell.py", line 2683, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\IPython\core\interactiveshell.py", line 2787, in run_ast_nodes
if self.run_code(code, result):
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\IPython\core\interactiveshell.py", line 2847, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-12-1e7a3b259f23>", line 4, in <module>
model.add(Dense(10, activation='softmax'))
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\models.py", line 466, in add
output_tensor = layer(self.outputs[0])
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\engine\topology.py", line 585, in __call__
output = self.call(inputs, **kwargs)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\layers\core.py", line 840, in call
output = K.dot(inputs, self.kernel)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\keras\backend\tensorflow_backend.py", line 936, in dot
out = tf.matmul(x, y)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\ops\math_ops.py", line 1801, in matmul
a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 1263, in _mat_mul
transpose_b=transpose_b, name=name)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 768, in apply_op
op_def=op_def)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\ops.py", line 2336, in create_op
original_op=self._default_original_op, op_def=op_def)
File "C:\Users\nicol\Anaconda3\envs\tensorflow\lib\site-packages\tensorflow\python\framework\ops.py", line 1228, in __init__
self._traceback = _extract_stack()
InternalError (see above for traceback): Blas GEMM launch failed : a.shape=(64, 784), b.shape=(784, 10), m=64, n=10, k=784
[[Node: dense_1/MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/gpu:0"](flatten_1/Reshape, dense_1/kernel/read)]]
[[Node: Assign_3/_84 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_374_Assign_3", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]]
In both cases, the error is with InternalError (see above for traceback): Blas GEMM launch failed Can you tell me how to get Blas GEMM to launch? I installed tensorflow and keras in a 3.5 python anaconda environment where I also installed all needed module (numpy, pandas, scipy, scikit-learn). I have a Windows 10 with a NVIDIA gpu that can use CUDA. I downloaded CUDA and cuDNN. I'm using the Jupyter notebook on Chrome.
Sometimes when I run my code, rather than having this error, I get that it starts running and then it crashes. After the crash, I can't do anything on my jupyter notebook and after some time a pop-up asks me if I want to kill the page. This is an image of what I got after the crash. !(http://www.hostingpics.net/viewer.php?id=647186tensorflowError.png)
P.S. I know my problem is similar as in this question: Tensorflow Basic Example Error: CUBLAS_STATUS_NOT_INITIALIZED but it has not been solved there and I'm not sure this question is clear enough or is exactly the same problem as I have so I'm posting it with my own error message. This problem is different of: TensorFlow: InternalError: Blas SGEMM launch failed Since I have a problem with GEMM rather than SGEMM and that my problem is both with gpu and cpu and it is not solved by the answer of this question.
Solution 1:
This worked for me on TensorFlow 2.1.0 (per: https://www.tensorflow.org/api_docs/python/tf/config/experimental/set_memory_growth)
import tensorflow as tf
physical_devices = tf.config.list_physical_devices('GPU')
for device in physical_devices:
tf.config.experimental.set_memory_growth(device, True)
Solution 2:
It's a simple fix, but it was a nightmare to figure it all out
On Windows I found the Keras install in Anaconda3\Lib\site-packages\keras
sources:
https://www.tensorflow.org/guide/using_gpu
https://github.com/keras-team/keras/blob/master/keras/backend/tensorflow_backend.py
Find the following in your keras/tensorflow_backend.py file you'll add config.gpu_options.allow_growth= True in both places
if _SESSION is None:
if not os.environ.get('OMP_NUM_THREADS'):
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth=True
else:
num_thread = int(os.environ.get('OMP_NUM_THREADS'))
config = tf.ConfigProto(intra_op_parallelism_threads=num_thread,
allow_soft_placement=True)
config.gpu_options.allow_growth=True
_SESSION = tf.Session(config=config)
session = _SESSION
Solution 3:
Make sure you have no other processes using the GPU running. Run nvidia-smi to check this.
SOURCE: An issue brought up by @reedwm.
Solution 4:
Adding the following lines after imports solved the problem:
configuration = tf.compat.v1.ConfigProto()
configuration.gpu_options.allow_growth = True
session = tf.compat.v1.Session(config=configuration)
Solution 5:
This Answer is much related to Tensorflow:
Sometimes Tensorflow fails at creation in Windows.
Restarting the notebook using gpu solves it in most cases
If it doesnt then try restarting the notebook after adding these options in your code.
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,allow_soft_placement=True)
I never had such error while using Keras But try restarting your notebook