yuusuke0706d
@yuusuke0706d (yu da)

Are you sure you want to delete the question?

If your question is resolved, you may close it.

Leaving a resolved question undeleted may help others!

We hope you find it useful!

Python エラー Unicode Python Machine Learning

解決したいこと

初心者です
Python Machine Learning: Machine Learning and Deep Learning with Python, scikit-learn, and TensorFlow 2, 3rd Editionのch14 part2 のコード
https://github.com/rasbt/python-machine-learning-book-3rd-edition

ユニコードエラーはわかるけどどこ直せばいいかわからない

発生している問題・エラー

---------------------------------------------------------------------------
UnicodeDecodeError                        Traceback (most recent call last)
<ipython-input-99-50fe8c199257> in <module>
      3 total_steps=EPOCHS*int(np.ceil(len(df_train)/BATCH_SIZE))
      4 print('Training Steps:',total_steps)
----> 5 regressor.train(input_fn=lambda:train_input_fn(df_train_norm,batch_size=BATCH_SIZE),
      6                steps=total_steps)

~\anaconda3\lib\site-packages\tensorflow_estimator\python\estimator\estimator.py in train(self, input_fn, hooks, steps, max_steps, saving_listeners)
    358 
    359       saving_listeners = _check_listeners_type(saving_listeners)
--> 360       loss = self._train_model(input_fn, hooks, saving_listeners)
    361       logging.info('Loss for final step: %s.', loss)
    362       return self

~\anaconda3\lib\site-packages\tensorflow_estimator\python\estimator\estimator.py in _train_model(self, input_fn, hooks, saving_listeners)
   1184       return self._train_model_distributed(input_fn, hooks, saving_listeners)
   1185     else:
-> 1186       return self._train_model_default(input_fn, hooks, saving_listeners)
   1187 
   1188   def _train_model_default(self, input_fn, hooks, saving_listeners):

~\anaconda3\lib\site-packages\tensorflow_estimator\python\estimator\estimator.py in _train_model_default(self, input_fn, hooks, saving_listeners)
   1215                                            self.config)
   1216       global_step_tensor = tf.compat.v1.train.get_global_step(g)
-> 1217       return self._train_with_estimator_spec(estimator_spec, worker_hooks,
   1218                                              hooks, global_step_tensor,
   1219                                              saving_listeners)

~\anaconda3\lib\site-packages\tensorflow_estimator\python\estimator\estimator.py in _train_with_estimator_spec(self, estimator_spec, worker_hooks, hooks, global_step_tensor, saving_listeners)
   1510                   output_dir=self._config.model_dir))
   1511 
-> 1512     with training.MonitoredTrainingSession(
   1513         master=self._config.master,
   1514         is_chief=self._config.is_chief,

~\anaconda3\lib\site-packages\tensorflow\python\training\monitored_session.py in MonitoredTrainingSession(master, is_chief, checkpoint_dir, scaffold, hooks, chief_only_hooks, save_checkpoint_secs, save_summaries_steps, save_summaries_secs, config, stop_grace_period_secs, log_step_count_steps, max_wait_secs, save_checkpoint_steps, summary_dir, save_graph_def)
    611   if hooks:
    612     all_hooks.extend(hooks)
--> 613   return MonitoredSession(
    614       session_creator=session_creator,
    615       hooks=all_hooks,

~\anaconda3\lib\site-packages\tensorflow\python\training\monitored_session.py in __init__(self, session_creator, hooks, stop_grace_period_secs)
   1056                hooks=None,
   1057                stop_grace_period_secs=120):
-> 1058     super(MonitoredSession, self).__init__(
   1059         session_creator,
   1060         hooks,

~\anaconda3\lib\site-packages\tensorflow\python\training\monitored_session.py in __init__(self, session_creator, hooks, should_recover, stop_grace_period_secs)
    759         stop_grace_period_secs=stop_grace_period_secs)
    760     if should_recover:
--> 761       self._sess = _RecoverableSession(self._coordinated_creator)
    762     else:
    763       self._sess = self._coordinated_creator.create_session()

~\anaconda3\lib\site-packages\tensorflow\python\training\monitored_session.py in __init__(self, sess_creator)
   1265     """
   1266     self._sess_creator = sess_creator
-> 1267     _WrappedSession.__init__(self, self._create_session())
   1268 
   1269   def _create_session(self):

~\anaconda3\lib\site-packages\tensorflow\python\training\monitored_session.py in _create_session(self)
   1270     while True:
   1271       try:
-> 1272         return self._sess_creator.create_session()
   1273       except _PREEMPTION_ERRORS as e:
   1274         logging.info(

~\anaconda3\lib\site-packages\tensorflow\python\training\monitored_session.py in create_session(self)
    919       # Inform the hooks that a new session has been created.
    920       for hook in self._hooks:
--> 921         hook.after_create_session(self.tf_sess, self.coord)
    922       return _CoordinatedSession(
    923           _HookedSession(self.tf_sess, self._hooks), self.coord,

~\anaconda3\lib\site-packages\tensorflow\python\training\basic_session_run_hooks.py in after_create_session(self, session, coord)
    601     self._summary_writer.add_meta_graph(meta_graph_def)
    602     # The checkpoint saved here is the state at step "global_step".
--> 603     self._save(session, global_step)
    604     self._timer.update_last_triggered_step(global_step)
    605 

~\anaconda3\lib\site-packages\tensorflow\python\training\basic_session_run_hooks.py in _save(self, session, step)
    633 
    634     logging.info("Saving checkpoints for %d into %s.", step, self._save_path)
--> 635     self._get_saver().save(session, self._save_path, global_step=step,
    636                            write_meta_graph=self._save_graph_def)
    637     self._summary_writer.add_session_log(

~\anaconda3\lib\site-packages\tensorflow\python\training\saver.py in save(self, sess, save_path, global_step, latest_filename, meta_graph_suffix, write_meta_graph, write_state, strip_default_attrs, save_debug_info)
   1274           model_checkpoint_path = self.saver_def.save_tensor_name
   1275         else:
-> 1276           model_checkpoint_path = sess.run(
   1277               self.saver_def.save_tensor_name,
   1278               {self.saver_def.filename_tensor_name: checkpoint_file})

~\anaconda3\lib\site-packages\tensorflow\python\client\session.py in run(self, fetches, feed_dict, options, run_metadata)
    968 
    969     try:
--> 970       result = self._run(None, fetches, feed_dict, options_ptr,
    971                          run_metadata_ptr)
    972       if run_metadata:

~\anaconda3\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
   1191     # or if the call is a partial run that specifies feeds.
   1192     if final_fetches or final_targets or (handle and feed_dict_tensor):
-> 1193       results = self._do_run(handle, final_targets, final_fetches,
   1194                              feed_dict_tensor, options, run_metadata)
   1195     else:

~\anaconda3\lib\site-packages\tensorflow\python\client\session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1371 
   1372     if handle is None:
-> 1373       return self._do_call(_run_fn, feeds, fetches, targets, options,
   1374                            run_metadata)
   1375     else:

~\anaconda3\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
   1378   def _do_call(self, fn, *args):
   1379     try:
-> 1380       return fn(*args)
   1381     except errors.OpError as e:
   1382       message = compat.as_text(e.message)

~\anaconda3\lib\site-packages\tensorflow\python\client\session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
   1361       # Ensure any changes to the graph are reflected in the runtime.
   1362       self._extend_graph()
-> 1363       return self._call_tf_sessionrun(options, feed_dict, fetch_list,
   1364                                       target_list, run_metadata)
   1365 

~\anaconda3\lib\site-packages\tensorflow\python\client\session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
   1454   def _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list,
   1455                           run_metadata):
-> 1456     return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
   1457                                             fetch_list, target_list,
   1458                                             run_metadata)

UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8e in position 153: invalid start byte

該当するソースコード


#df_train_norm  に関わる
dataset_path = tf.keras.utils.get_file("auto-mpg.data", 
                                       ("http://archive.ics.uci.edu/ml/machine-learning-databases"
                                        "/auto-mpg/auto-mpg.data"))

column_names = ['MPG', 'Cylinders', 'Displacement', 'Horsepower',
                'Weight', 'Acceleration', 'ModelYear', 'Origin']

df = pd.read_csv(dataset_path, names=column_names,
                 na_values = "?", comment='\t',
                 sep=" ", skipinitialspace=True)

df = df.dropna()
df = df.reset_index(drop=True)

import sklearn
import sklearn.model_selection


df_train, df_test = sklearn.model_selection.train_test_split(df, train_size=0.8)
train_stats = df_train.describe().transpose()

numeric_column_names = ['Cylinders', 'Displacement', 'Horsepower', 'Weight', 'Acceleration']

df_train_norm, df_test_norm = df_train.copy(), df_test.copy()

for col_name in numeric_column_names:
    mean = train_stats.loc[col_name, 'mean']
    std  = train_stats.loc[col_name, 'std']
    df_train_norm.loc[:, col_name] = (df_train_norm.loc[:, col_name] - mean)/std
    df_test_norm.loc[:, col_name] = (df_test_norm.loc[:, col_name] - mean)/std

#エラーが出た点
EPOCHS = 1000
BATCH_SIZE = 8
total_steps = EPOCHS * int(np.ceil(len(df_train) / BATCH_SIZE))
print('Training Steps:', total_steps)

regressor.train(
    input_fn=lambda:train_input_fn(df_train_norm, batch_size=BATCH_SIZE),
    steps=total_steps)

自分で試したこと

3行目のpd.read_csvの引数encodingにshift_jisとutf-8を入れてみた、ダメだった

0

3Answer

This answer has been deleted for violation of our Terms of Service.

This answer has been deleted for violation of our Terms of Service.

 次の修正により、問題を解決することができたので、お知らせします。

 この質問には、ソースとして含まれていませんが、regressor の定義において、model_dirを指定する引数の記述に問題があります。(書籍及び原書並びに次のURLのサンプルコード)
https://github.com/rasbt/python-machine-learning-book-3rd-edition/blob/master/ch14/ch14_part2.ipynb

regressor = tf.estimator.DNNRegressor(
feature_columns=all_feature_columns,
hidden_units=[32, 10],
model_dir='models/autompg-dnnregressor/')

 スラッシュ記号によりディレクトリ名の切れ目(2か所)が記述されていますが、これを\に変更すると、良好な結果が得られます。

0Like

Comments

  1.  '\' は、半角の¥¥です。
     この対策は、「解決」と言えるか否か不明ですが、とりあえず動きます。
     「解決」と言えないかも知れないと思う理由は、本来APレベルで、OSに
    依存して区々の書き方を要請されるというのは、前時代のような気がする
    からです。OSと言語系の間の調整は、APから見えないことが理想ではない
    かというのが私の考えです。
     ですから、Windows側の環境変数 (PYTHONUTF8 1)が効果的であるとか、
    Python実行時のオプション(X -utf8)が効果的であるとありがたいと思う
    次第。現時点で、いずれも成功させることができていません。これは、こ
    れで問題なので、なんとか解決したいと思いつつも、時間が足りない。

Your answer might help someone💌