class BaseModel(Model):\n\n def\
- \ __init__(self, base_network):\n super(BaseModel, self).__init__()\n self.network\
- \ = base_network\n \n def call(self, inputs):\n print(inputs)\n return\
- \ self.network(inputs)\n\ndef get_base_model():\n inputs = tf.keras.Input(shape=INPUT)\n\
- \n conv2d_1 = layers.Conv2D(name='seq_1', filters=64, \n kernel_size=20,\
- \ \n activation='relu')(inputs)\n maxpool_1 = layers.MaxPooling2D(pool_size=(2,\
- \ 2))(conv2d_1)\n\n conv2d_2 = layers.Conv2D(filters=128, \n kernel_size=20,\
- \ \n activation='relu')(maxpool_1)\n maxpool_2 = layers.MaxPooling2D(pool_size=(2,\
- \ 2))(conv2d_2)\n\n conv2d_3 = layers.Conv2D(filters=128, \n kernel_size=20,\
- \ \n activation='relu')(maxpool_2)\n maxpool_3 = layers.MaxPooling2D(pool_size=(2,\
- \ 2))(conv2d_3)\n\n conv2d_4 = layers.Conv2D(filters=256, \n kernel_size=10,\
- \ \n activation='relu')(maxpool_3)\n\n flatten_1 = layers.Flatten()(conv2d_4)\n\
- \ outputs = layers.Dense(units=4096,\n activation='sigmoid')(flatten_1)\n\
- \ \n model = Model(inputs=inputs, outputs=outputs)\n\n return model\n\n\
- Then, I'm building the Siamese network using the previous method like that:
\n\ -INPUT = (250, 250, 3)\n\ndef\
- \ get_siamese_model():\n left_input = layers.Input(name='img1', shape=INPUT)\n\
- \ right_input = layers.Input(name='img2', shape=INPUT)\n \n base_model = get_base_model()\n\
- \ base_model = BaseModel(base_model)\n\n # bind the two input layers to the\
- \ base network\n left = base_model(left_input)\n right = base_model(right_input)\n\
- \n # build distance measuring layer\n l1_lambda = layers.Lambda(lambda tensors:abs(tensors[0]\
- \ - tensors[1]))\n l1_dist = l1_lambda([left, right])\n\n pred = layers.Dense(1,activation='sigmoid')(l1_dist)\n\
- \n return Model(inputs=[left_input, right_input], outputs=pred)\n\nclass SiameseNetwork(Model):\n\
- \n def __init__(self, siamese_network):\n super(SiameseNetwork, self).__init__()\n\
- \ self.siamese_network = siamese_network\n \n def call(self, inputs):\n \
- \ print(inputs)\n return self.siamese_network(inputs)\nI'm\
- \ then training the network by passing a tf.data.Dataset to it:
net.fit(x=train_dataset, epochs=10\
- \ ,verbose=True)\ntrain_dataset is of type:
\n\n<PrefetchDataset shapes: ((None, 250, 250, 3), (None, 250,\ - \ 250, 3)), types: (tf.float32, tf.float32)>
\n
It seems\ - \ like the shape of the input is defined well, but I'm still encountering an error:
\n\ -ValueError \
- \ Traceback (most recent call last)\n<ipython-input-144-6c5586e1e205>\
- \ in <module>()\n----> 1 net.fit(x=train_dataset, epochs=10 ,verbose=True)\n\
- \n9 frames\n/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py\
- \ in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split,\
- \ validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch,\
- \ validation_steps, validation_batch_size, validation_freq, max_queue_size, workers,\
- \ use_multiprocessing)\n 1098 _r=1):\n 1099 \
- \ callbacks.on_train_batch_begin(step)\n-> 1100 tmp_logs = self.train_function(iterator)\n\
- \ 1101 if data_handler.should_sync:\n 1102 context.async_wait()\n\
- \n/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\
- \ in __call__(self, *args, **kwds)\n 826 tracing_count = self.experimental_get_tracing_count()\n\
- \ 827 with trace.Trace(self._name) as tm:\n--> 828 result = self._call(*args,\
- \ **kwds)\n 829 compiler = "xla" if self._experimental_compile\
- \ else "nonXla"\n 830 new_tracing_count = self.experimental_get_tracing_count()\n\
- \n/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\
- \ in _call(self, *args, **kwds)\n 869 # This is the first call of __call__,\
- \ so we have to initialize.\n 870 initializers = []\n--> 871 \
- \ self._initialize(args, kwds, add_initializers_to=initializers)\n 872 \
- \ finally:\n 873 # At this point we know that the initialization is\
- \ complete (or less\n\n/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\
- \ in _initialize(self, args, kwds, add_initializers_to)\n 724 self._concrete_stateful_fn\
- \ = (\n 725 self._stateful_fn._get_concrete_function_internal_garbage_collected(\
- \ # pylint: disable=protected-access\n--> 726 *args, **kwds))\n\
- \ 727 \n 728 def invalid_creator_scope(*unused_args, **unused_kwds):\n\
- \n/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py in\
- \ _get_concrete_function_internal_garbage_collected(self, *args, **kwargs)\n \
- \ 2967 args, kwargs = None, None\n 2968 with self._lock:\n-> 2969\
- \ graph_function, _ = self._maybe_define_function(args, kwargs)\n 2970\
- \ return graph_function\n 2971 \n\n/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\
- \ in _maybe_define_function(self, args, kwargs)\n 3359 \n 3360 self._function_cache.missed.add(call_context_key)\n\
- -> 3361 graph_function = self._create_graph_function(args, kwargs)\n\
- \ 3362 self._function_cache.primary[cache_key] = graph_function\n\
- \ 3363 \n\n/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\
- \ in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)\n \
- \ 3204 arg_names=arg_names,\n 3205 override_flat_arg_shapes=override_flat_arg_shapes,\n\
- -> 3206 capture_by_value=self._capture_by_value),\n 3207 \
- \ self._function_attributes,\n 3208 function_spec=self.function_spec,\n\
- \n/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py\
- \ in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph,\
- \ autograph, autograph_options, add_control_dependencies, arg_names, op_return_value,\
- \ collections, capture_by_value, override_flat_arg_shapes)\n 988 _,\
- \ original_func = tf_decorator.unwrap(python_func)\n 989 \n--> 990 \
- \ func_outputs = python_func(*func_args, **func_kwargs)\n 991 \n 992 \
- \ # invariant: `func_outputs` contains only Tensors, CompositeTensors,\n\n\
- /usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\
- \ in wrapped_fn(*args, **kwds)\n 632 xla_context.Exit()\n 633\
- \ else:\n--> 634 out = weak_wrapped_fn().__wrapped__(*args,\
- \ **kwds)\n 635 return out\n 636 \n\n/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py\
- \ in wrapper(*args, **kwargs)\n 975 except Exception as e: # pylint:disable=broad-except\n\
- \ 976 if hasattr(e, "ag_error_metadata"):\n--> 977\
- \ raise e.ag_error_metadata.to_exception(e)\n 978 \
- \ else:\n 979 raise\n\nValueError: in user code:\n\n /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:805\
- \ train_function *\n return step_function(self, iterator)\n <ipython-input-125-de3a74f810c3>:9\
- \ call *\n return self.siamese_network(inputs)\n /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/base_layer.py:998\
- \ __call__ **\n input_spec.assert_input_compatibility(self.input_spec,\
- \ inputs, self.name)\n /usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/input_spec.py:207\
- \ assert_input_compatibility\n ' input tensors. Inputs received: ' + str(inputs))\n\
- \n ValueError: Layer model_16 expects 2 input(s), but it received 1 input tensors.\
- \ Inputs received: [<tf.Tensor 'IteratorGetNext:0' shape=(None, 250, 250, 3)\
- \ dtype=float32>]\nI do undertand that model_16\
- \ is the BaseModel, however I can't figure out what am I doing wrong here.
New to TensorFlow, so apologies for newbie question.
\n\nFollowing\ - \ this tutorial but instead of using image data I am\ - \ using numerical data.
\n\nLoad the dataset:
\n\ntrain_dataset_url\
- \ = \"xxx.csv\"\ntrain_dataset_fp = tf.keras.utils.get_file(\n fname=os.path.basename(train_dataset_url),\n\
- \ origin=train_dataset_url)\nMake training dataset:
\n\ - \nbatch_size = 32\n\ntrain_dataset = tf.contrib.data.make_csv_dataset(\n\
- \ train_dataset_fp,\n batch_size, \n column_names=column_names,\n \
- \ label_name=label_name,\n num_epochs=1)\nTrain\ - \ classified model using:
\n\nmodel = tf.keras.Sequential([\n\
- \ tf.keras.layers.Dense(10, activation=tf.nn.relu, input_shape=(1,)),\n tf.keras.layers.Dense(10,\
- \ activation=tf.nn.relu),\n tf.keras.layers.Dense(4)\n])
But\ - \ when I \"test\" the model with the same inputs:
\n\npredictions\
- \ = model(features)
I receive the error:
\n\nInvalidArgumentError:\
- \ cannot compute MatMul as input #0(zero-based) was expected to be a float tensor\
- \ but is a int32 tensor [Op:MatMul]
It's possible I have missed\ - \ something fundamental. I feel like I need to specify a type somewhere.
\n" -- text: "I have created a neural style transfer with Eager Execution, but it does\ - \ not work when I try to turn it into a tf.function.\nThe error message says:
\n\ -ValueError: tf.function only supports singleton tf.Variables created\
- \ on the first call. Make sure the tf.Variable is only created once or created\
- \ outside tf.function. See https://www.tensorflow.org/guide/function#creating_tfvariables\
- \ for more information.\nHowever, no variable is being created\ - \ inside the function. Here is a simplified version of the code, which is just\ - \ a neural style transfer with one image (the goal is to make the generated image\ - \ look exactly like the content image):
\nimport tensorflow as tf\n\
- import numpy as np\nfrom PIL import Image\n\n#Get and process the images\nimage\
- \ = np.array(Image.open("frame7766.jpg")).reshape(1, 720, 1280, 3)/255\n\
- content_image = tf.convert_to_tensor(image, dtype = tf.float32)\n# variable is\
- \ defined outside of tf.function\ngenerated_image = tf.Variable(np.random.rand(1,\
- \ 720, 1280, 3)/2 + content_image/2, dtype = tf.float32)\n\ndef clip_0_1(image):\
- \ # keeps image values between 0 and 1\n return tf.clip_by_value(image, clip_value_min=0,\
- \ clip_value_max=1)\n\n@ tf.function\ndef train_step(generated_image, content_image):\
- \ #turn generated image into tf variable\n optimizer = tf.keras.optimizers.Adam(learning_rate\
- \ = 0.01)\n with tf.GradientTape() as tape:\n cost = tf.reduce_sum(tf.square(generated_image\
- \ - content_image))\n grad = tape.gradient(cost, generated_image) \n optimizer.apply_gradients([(grad,\
- \ generated_image)]) # More information below\n generated_image.assign(clip_0_1(generated_image))\n\
- \ return generated_image\n\ngenerated_image = train_step(generated_image, content_image)\n\
- The error message points to the line
\noptimizer.apply_gradients([(grad,\
- \ generated_image)]) \nI have tried to change the input of \
- \ optimizer.apply_gradients to zip([grad], [generated_image]),\
- \ and every combination of lists and tuples I can think of, but the error still\
- \ remains. I have also looked through https://www.tensorflow.org/guide/function#creating_tfvariables\
- \ and https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Optimizer,\
- \ but neither of them shows examples where the variable is not explicitly defined.\n\
- The only conclusion that I can come to is that one of my commands (most likely\
- \ optimizer.apply_gradients) creates a variable because of an issue\
- \ in my earlier code. Is that correct?
I am going through TensorFlow Eager Execution from here and find it difficult to understand the\ - \ customizing gradients part.
\n\n@tfe.custom_gradient\ndef logexp(x):\n\
- \ e = tf.exp(x)\n def grad(dy):\n return dy * (1 - 1/(1 + e))\n \
- \ return tf.log(1 + e), grad\nFirst, it is difficult to\ - \ make sense what does dy do in the gradient function.
\n\nWhen I read the\ - \ implementation of tf.contrib.eager.custom_gradient.\nI can't really make sense\ - \ the working mechanism behind tape. Following is the code I borrow from the implementation\ - \ of tf.contrib.eager.custom_gradient. Can anybody explain what does tape do here?
\n\ - \nfrom tensorflow.python.eager import tape\nfrom tensorflow.python.ops\
- \ import array_ops\nfrom tensorflow.python.ops import gen_array_ops\nfrom tensorflow.python.util\
- \ import nest\nfrom tensorflow.python.framework import ops as tf_ops\n\ndef my_custom_gradient(f):\n\
- \ def decorated(*args, **kwargs):\n for x in args:\n print('args\
- \ {0}'.format(x))\n input_tensors = [tf_ops.convert_to_tensor(x) for x\
- \ in args]\n\n with tape.stop_recording():\n result, grad_fn\
- \ = f(*args, **kwargs)\n flat_result = nest.flatten(result)\n \
- \ flat_result = [gen_array_ops.identity(x) for x in flat_result]\n\n \
- \ def actual_grad_fn(*outputs):\n print(*outputs)\n \
- \ return nest.flatten(grad_fn(*outputs))\n\n tape.record_operation(\n \
- \ f.__name__, # the name of f, in this case logexp\n flat_result,\n\
- \ input_tensors,\n actual_grad_fn) # backward_function\n\
- \ flat_result = list(flat_result)\n return nest.pack_sequence_as(result,\
- \ flat_result)\nreturn decorated \nEven though I found the\ - \ implementation of tape from here. But I can't really get much out of it\ - \ due the poor documentation.
\n" -- text: "I want to extract signals from time series data for machine learning using\ - \ tensorflow. I got this error when I try to run the program.
\n\n\ -\n/Users/renzha/Library/Application Support/JetBrains/PyCharmCE2021.1/scratches/pywavelet.py:38:\ - \ DeprecationWarning:
\nnp.floatis a deprecated alias for the builtin\ - \float. To silence this warning, usefloatby itself.\ - \ Doing this will not modify any behavior and is safe. If you specifically wanted\ - \ the numpy scalar type, usenp.float64here.\nDeprecated in NumPy\ - \ 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n\ - X = np.asarray(X).astype(np.float)\nTraceback (most recent call last):\nFile "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/core/series.py",\ - \ line 129, in wrapper\nraise TypeError(f"cannot convert the series to {converter}")\n\ - TypeError: cannot convert the series to <class 'float'>\nThe above exception\ - \ was the direct cause of the following exception:\nTraceback (most recent call\ - \ last):\nFile "/Users/renzha/Library/Application Support/JetBrains/PyCharmCE2021.1/scratches/pywavelet.py",\ - \ line 38, in \nX = np.asarray(X).astype(np.float)\nValueError: setting an array\ - \ element with a sequence.
However, if I use X =\
- \ np.array(X), then the error will be
\n\nValueError:\ - \ Failed to convert a NumPy array to a Tensor (Unsupported object type Series).
\n\ -
I have tried tf.convert_to_tensor(X), but it will\
- \ return the same error.
The code is here:
\nHBT_data =\
- \ []\nfor i in range (0,10):\n files = glob.glob('/Users/renzha/Documents/work/preparing\
- \ papers/non-contact HCI/s letter' + str(i) +'/*')\n\n for file in files:\n\
- \ names = ['time', 'signal']\n data = pd.read_csv (file, names=names)\n\
- \ data = data.drop (data.index [0])\n data = data.dropna (axis=0,\
- \ how='any')\n x = data.iloc[:,1]\n filename = os.path.basename\
- \ (file)\n label = os.path.splitext(filename)[0]\n labeledArray\
- \ = [label, x]\n HBT_data.append(labeledArray)\n\nHBT_data = pd.DataFrame(HBT_data,\
- \ dtype=object)\ny = HBT_data.iloc[:, 0]\ny = np.asarray(y)\nX = HBT_data.iloc[:,\
- \ 1:]\nX = np.asarray(X).astype(np.float) # tried but did not work\nX = tf.convert_to_tensor\
- \ (X) # tried but did not work\n\nX_train, X_validation, Y_train, Y_validation\
- \ = train_test_split(X, y, test_size=0.2, random_state=8)\n\nX_train = X_train.reshape((X_train.shape[0],\
- \ X_train.shape[1], 1))\nX_test = X_validation.reshape((X_validation.shape[0],\
- \ X_validation.shape[1], 1))\nnum_classes = len(np.unique(Y_train))\n\nidx = np.random.permutation(len(X_train))\n\
- x_train = X_train[idx]\ny_train = X_train[idx]\n\nY_train[Y_train == -1] = 0\n\
- Y_validation[Y_validation == -1] = 0\n\n\ndef make_model(input_shape):\n input_layer\
- \ = keras.layers.Input(input_shape)\n\n conv1 = keras.layers.Conv1D(filters=64,\
- \ kernel_size=3, padding="same")(input_layer)\n conv1 = keras.layers.BatchNormalization()(conv1)\n\
- \ conv1 = keras.layers.ReLU()(conv1)\n\n conv2 = keras.layers.Conv1D(filters=64,\
- \ kernel_size=3, padding="same")(conv1)\n conv2 = keras.layers.BatchNormalization()(conv2)\n\
- \ conv2 = keras.layers.ReLU()(conv2)\n\n conv3 = keras.layers.Conv1D(filters=64,\
- \ kernel_size=3, padding="same")(conv2)\n conv3 = keras.layers.BatchNormalization()(conv3)\n\
- \ conv3 = keras.layers.ReLU()(conv3)\n\n gap = keras.layers.GlobalAveragePooling1D()(conv3)\n\
- \n output_layer = keras.layers.Dense(num_classes, activation="softmax")(gap)\n\
- \n return keras.models.Model(inputs=input_layer, outputs=output_layer)\n\n\n\
- model = make_model(input_shape=X_train.shape[1:])\nkeras.utils.plot_model(model,\
- \ show_shapes=True)\n\nepochs = 500\nbatch_size = 32\n\ncallbacks = [\n keras.callbacks.ModelCheckpoint(\n\
- \ "best_model.h5", save_best_only=True, monitor="val_loss"\n\
- \ ),\n keras.callbacks.ReduceLROnPlateau(\n monitor="val_loss",\
- \ factor=0.5, patience=20, min_lr=0.0001\n ),\n keras.callbacks.EarlyStopping(monitor="val_loss",\
- \ patience=50, verbose=1),\n]\nmodel.compile(\n optimizer="adam",\n\
- \ loss="sparse_categorical_crossentropy",\n metrics=["sparse_categorical_accuracy"],\n\
- )\nhistory = model.fit(\n X_train,\n Y_train,\n batch_size=batch_size,\n\
- \ epochs=epochs,\n callbacks=callbacks,\n validation_split=0.2,\n \
- \ verbose=1,\n)\n\nmodel = keras.models.load_model("best_model.h5")\n\
- \ntest_loss, test_acc = model.evaluate(X_validation, Y_validation)\n\nprint("Test\
- \ accuracy", test_acc)\nprint("Test loss", test_loss)```\n\nhttps://kwotsin.github.io/tech/2017/02/11/transfer-learning.html\n\ + I followed the above link to make a image classifier
\n\nTraining code:
\n\ + \nslim = tf.contrib.slim\n\ndataset_dir = './data'\nlog_dir = './log'\n\
+ checkpoint_file = './inception_resnet_v2_2016_08_30.ckpt'\nimage_size = 299\n\
+ num_classes = 21\nvlabels_file = './labels.txt'\nlabels = open(labels_file, 'r')\n\
+ labels_to_name = {}\nfor line in labels:\n label, string_name = line.split(':')\n\
+ \ string_name = string_name[:-1]\n labels_to_name[int(label)] = string_name\n\
+ \nfile_pattern = 'test_%s_*.tfrecord'\n\nitems_to_descriptions = {\n 'image':\
+ \ 'A 3-channel RGB coloured product image',\n 'label': 'A label that from 20\
+ \ labels'\n}\n\nnum_epochs = 10\nbatch_size = 16\ninitial_learning_rate = 0.001\n\
+ learning_rate_decay_factor = 0.7\nnum_epochs_before_decay = 4\n\ndef get_split(split_name,\
+ \ dataset_dir, file_pattern=file_pattern, file_pattern_for_counting='products'):\n\
+ \ if split_name not in ['train', 'validation']:\n raise ValueError(\n\
+ \ 'The split_name %s is not recognized. Please input either train or\
+ \ validation as the split_name' % (\n split_name))\n\n file_pattern_path\
+ \ = os.path.join(dataset_dir, file_pattern % (split_name))\n\n num_samples\
+ \ = 0\n file_pattern_for_counting = file_pattern_for_counting + '_' + split_name\n\
+ \ tfrecords_to_count = [os.path.join(dataset_dir, file) for file in os.listdir(dataset_dir)\
+ \ if\n file.startswith(file_pattern_for_counting)]\n\
+ \ for tfrecord_file in tfrecords_to_count:\n for record in tf.python_io.tf_record_iterator(tfrecord_file):\n\
+ \ num_samples += 1\n\n test = num_samples\n\n reader = tf.TFRecordReader\n\
+ \n keys_to_features = {\n 'image/encoded': tf.FixedLenFeature((), tf.string,\
+ \ default_value=''),\n 'image/format': tf.FixedLenFeature((), tf.string,\
+ \ default_value='jpg'),\n 'image/class/label': tf.FixedLenFeature(\n \
+ \ [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),\n }\n\
+ \n items_to_handlers = {\n 'image': slim.tfexample_decoder.Image(),\n\
+ \ 'label': slim.tfexample_decoder.Tensor('image/class/label'),\n }\n\
+ \n decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers)\n\
+ \n labels_to_name_dict = labels_to_name\n\n dataset = slim.dataset.Dataset(\n\
+ \ data_sources=file_pattern_path,\n decoder=decoder,\n reader=reader,\n\
+ \ num_readers=4,\n num_samples=num_samples,\n num_classes=num_classes,\n\
+ \ labels_to_name=labels_to_name_dict,\n items_to_descriptions=items_to_descriptions)\n\
+ \n return dataset\n\ndef load_batch(dataset, batch_size, height=image_size,\
+ \ width=image_size, is_training=True):\n '''\n Loads a batch for training.\n\
+ \n INPUTS:\n - dataset(Dataset): a Dataset class object that is created\
+ \ from the get_split function\n - batch_size(int): determines how big of a\
+ \ batch to train\n - height(int): the height of the image to resize to during\
+ \ preprocessing\n - width(int): the width of the image to resize to during\
+ \ preprocessing\n - is_training(bool): to determine whether to perform a training\
+ \ or evaluation preprocessing\n\n OUTPUTS:\n - images(Tensor): a Tensor\
+ \ of the shape (batch_size, height, width, channels) that contain one batch of\
+ \ images\n - labels(Tensor): the batch's labels with the shape (batch_size,)\
+ \ (requires one_hot_encoding).\n\n '''\n # First create the data_provider\
+ \ object\n data_provider = slim.dataset_data_provider.DatasetDataProvider(\n\
+ \ dataset,\n common_queue_capacity=24 + 3 * batch_size,\n \
+ \ common_queue_min=24)\n\n # Obtain the raw image using the get method\n \
+ \ raw_image, label = data_provider.get(['image', 'label'])\n\n # Perform\
+ \ the correct preprocessing for this image depending if it is training or evaluating\n\
+ \ image = inception_preprocessing.preprocess_image(raw_image, height, width,\
+ \ is_training)\n\n # As for the raw images, we just do a simple reshape to\
+ \ batch it up\n raw_image = tf.expand_dims(raw_image, 0)\n raw_image = tf.image.resize_nearest_neighbor(raw_image,\
+ \ [height, width])\n raw_image = tf.squeeze(raw_image)\n\n # Batch up the\
+ \ image by enqueing the tensors internally in a FIFO queue and dequeueing many\
+ \ elements with tf.train.batch.\n images, raw_images, labels = tf.train.batch(\n\
+ \ [image, raw_image, label],\n batch_size=batch_size,\n num_threads=4,\n\
+ \ capacity=4 * batch_size,\n allow_smaller_final_batch=True)\n\n\
+ \ return images, raw_images, labels\n\n\ndef run():\n # Create the log directory\
+ \ here. Must be done here otherwise import will activate this unneededly.\n \
+ \ if not os.path.exists(log_dir):\n os.mkdir(log_dir)\n\n # =======================\
+ \ TRAINING PROCESS =========================\n # Now we start to construct\
+ \ the graph and build our model\n with tf.Graph().as_default() as graph:\n\
+ \ tf.logging.set_verbosity(tf.logging.INFO) # Set the verbosity to INFO\
+ \ level\n\n # First create the dataset and load one batch\n dataset\
+ \ = get_split('train', dataset_dir, file_pattern=file_pattern)\n images,\
+ \ _, labels = load_batch(dataset, batch_size=batch_size)\n\n # Know the\
+ \ number steps to take before decaying the learning rate and batches per epoch\n\
+ \ num_batches_per_epoch = int(dataset.num_samples / batch_size)\n \
+ \ num_steps_per_epoch = num_batches_per_epoch # Because one step is one batch\
+ \ processed\n decay_steps = int(num_epochs_before_decay * num_steps_per_epoch)\n\
+ \n # Create the model inference\n with slim.arg_scope(inception_resnet_v2_arg_scope()):\n\
+ \ logits, end_points = inception_resnet_v2(images, num_classes=dataset.num_classes,\
+ \ is_training=True)\n\n # Define the scopes that you want to exclude for\
+ \ restoration\n exclude = ['InceptionResnetV2/Logits', 'InceptionResnetV2/AuxLogits']\n\
+ \ variables_to_restore = slim.get_variables_to_restore(exclude=exclude)\n\
+ \n # Perform one-hot-encoding of the labels (Try one-hot-encoding within\
+ \ the load_batch function!)\n one_hot_labels = slim.one_hot_encoding(labels,\
+ \ dataset.num_classes)\n\n # Performs the equivalent to tf.nn.sparse_softmax_cross_entropy_with_logits\
+ \ but enhanced with checks\n loss = tf.losses.softmax_cross_entropy(onehot_labels=one_hot_labels,\
+ \ logits=logits)\n total_loss = tf.losses.get_total_loss() # obtain the\
+ \ regularization losses as well\n\n # Create the global step for monitoring\
+ \ the learning_rate and training.\n global_step = get_or_create_global_step()\n\
+ \n # Define your exponentially decaying learning rate\n lr = tf.train.exponential_decay(\n\
+ \ learning_rate=initial_learning_rate,\n global_step=global_step,\n\
+ \ decay_steps=decay_steps,\n decay_rate=learning_rate_decay_factor,\n\
+ \ staircase=True)\n\n # Now we can define the optimizer that\
+ \ takes on the learning rate\n optimizer = tf.train.AdamOptimizer(learning_rate=lr)\n\
+ \n # Create the train_op.\n train_op = slim.learning.create_train_op(total_loss,\
+ \ optimizer)\n\n # State the metrics that you want to predict. We get a\
+ \ predictions that is not one_hot_encoded.\n predictions = tf.argmax(end_points['Predictions'],\
+ \ 1)\n probabilities = end_points['Predictions']\n accuracy, accuracy_update\
+ \ = tf.contrib.metrics.streaming_accuracy(predictions, labels)\n metrics_op\
+ \ = tf.group(accuracy_update, probabilities)\n\n # Now finally create all\
+ \ the summaries you need to monitor and group them into one summary op.\n \
+ \ tf.summary.scalar('losses/Total_Loss', total_loss)\n tf.summary.scalar('accuracy',\
+ \ accuracy)\n tf.summary.scalar('learning_rate', lr)\n my_summary_op\
+ \ = tf.summary.merge_all()\n\n # Now we need to create a training step\
+ \ function that runs both the train_op, metrics_op and updates the global_step\
+ \ concurrently.\n def train_step(sess, train_op, global_step):\n \
+ \ '''\n Simply runs a session for the three arguments provided\
+ \ and gives a logging on the time elapsed for each global step\n '''\n\
+ \ # Check the time for each sess run\n start_time = time.time()\n\
+ \ total_loss, global_step_count, _ = sess.run([train_op, global_step,\
+ \ metrics_op])\n time_elapsed = time.time() - start_time\n\n \
+ \ # Run the logging to print some results\n logging.info('global\
+ \ step %s: loss: %.4f (%.2f sec/step)', global_step_count, total_loss, time_elapsed)\n\
+ \n return total_loss, global_step_count\n\n # Now we create\
+ \ a saver function that actually restores the variables from a checkpoint file\
+ \ in a sess\n saver = tf.train.Saver(variables_to_restore)\n\n def\
+ \ restore_fn(sess):\n return saver.restore(sess, checkpoint_file)\n\
+ \n # Define your supervisor for running a managed session. Do not run the\
+ \ summary_op automatically or else it will consume too much memory\n sv\
+ \ = tf.train.Supervisor(logdir=log_dir, summary_op=None, init_fn=restore_fn)\n\
+ \n # Run the managed session\n with sv.managed_session() as sess:\n\
+ \ for step in xrange(num_steps_per_epoch * num_epochs):\n \
+ \ # At the start of every epoch, show the vital information:\n \
+ \ if step % num_batches_per_epoch == 0:\n logging.info('Epoch\
+ \ %s/%s', step / num_batches_per_epoch + 1, num_epochs)\n learning_rate_value,\
+ \ accuracy_value = sess.run([lr, accuracy])\n logging.info('Current\
+ \ Learning Rate: %s', learning_rate_value)\n logging.info('Current\
+ \ Streaming Accuracy: %s', accuracy_value)\n\n # optionally,\
+ \ print your logits and predictions for a sanity check that things are going fine.\n\
+ \ logits_value, probabilities_value, predictions_value, labels_value\
+ \ = sess.run(\n [logits, probabilities, predictions, labels])\n\
+ \ print 'logits: \\n', logits_value\n print\
+ \ 'Probabilities: \\n', probabilities_value\n print 'predictions:\
+ \ \\n', predictions_value\n print 'Labels:\\n:', labels_value\n\
+ \n # Log the summaries every 10 step.\n if step\
+ \ % 10 == 0:\n loss, _ = train_step(sess, train_op, sv.global_step)\n\
+ \ summaries = sess.run(my_summary_op)\n \
+ \ sv.summary_computed(sess, summaries)\n\n # If not, simply run\
+ \ the training step\n else:\n loss, _ = train_step(sess,\
+ \ train_op, sv.global_step)\n\n # We log the final training loss and\
+ \ accuracy\n logging.info('Final Loss: %s', loss)\n logging.info('Final\
+ \ Accuracy: %s', sess.run(accuracy))\n\n # Once all the training has\
+ \ been done, save the log files and checkpoint model\n logging.info('Finished\
+ \ training! Saving model to disk now.')\n sv.saver.save(sess, sv.save_path,\
+ \ global_step=sv.global_step)\nThis code seems to work an\ + \ I have ran training on some sample data and Im getting 94% accuracy
\n\n\ +Evaluation code:
\n\nlog_dir = './log'\nlog_eval = './log_eval_test'\n\
+ dataset_dir = './data'\nbatch_size = 10\nnum_epochs = 1\n\ncheckpoint_file = tf.train.latest_checkpoint('./')\n\
+ \n\ndef run():\n if not os.path.exists(log_eval):\n os.mkdir(log_eval)\n\
+ \ with tf.Graph().as_default() as graph:\n tf.logging.set_verbosity(tf.logging.INFO)\n\
+ \ dataset = get_split('train', dataset_dir)\n images, raw_images,\
+ \ labels = load_batch(dataset, batch_size=batch_size, is_training=False)\n\n \
+ \ num_batches_per_epoch = dataset.num_samples / batch_size\n num_steps_per_epoch\
+ \ = num_batches_per_epoch\n\n with slim.arg_scope(inception_resnet_v2_arg_scope()):\n\
+ \ logits, end_points = inception_resnet_v2(images, num_classes=dataset.num_classes,\
+ \ is_training=False)\n\n variables_to_restore = slim.get_variables_to_restore()\n\
+ \ saver = tf.train.Saver(variables_to_restore)\n\n def restore_fn(sess):\n\
+ \ return saver.restore(sess, checkpoint_file)\n\n predictions\
+ \ = tf.argmax(end_points['Predictions'], 1)\n accuracy, accuracy_update\
+ \ = tf.contrib.metrics.streaming_accuracy(predictions, labels)\n metrics_op\
+ \ = tf.group(accuracy_update)\n\n global_step = get_or_create_global_step()\n\
+ \ global_step_op = tf.assign(global_step, global_step + 1)\n\n def\
+ \ eval_step(sess, metrics_op, global_step):\n '''\n Simply\
+ \ takes in a session, runs the metrics op and some logging information.\n \
+ \ '''\n start_time = time.time()\n _, global_step_count,\
+ \ accuracy_value = sess.run([metrics_op, global_step_op, accuracy])\n \
+ \ time_elapsed = time.time() - start_time\n\n logging.info('Global\
+ \ Step %s: Streaming Accuracy: %.4f (%.2f sec/step)', global_step_count, accuracy_value,\n\
+ \ time_elapsed)\n\n return accuracy_value\n\
+ \n tf.summary.scalar('Validation_Accuracy', accuracy)\n my_summary_op\
+ \ = tf.summary.merge_all()\n\n sv = tf.train.Supervisor(logdir=log_eval,\
+ \ summary_op=None, saver=None, init_fn=restore_fn)\n\n with sv.managed_session()\
+ \ as sess:\n for step in xrange(num_steps_per_epoch * num_epochs):\n\
+ \ sess.run(sv.global_step)\n if step % num_batches_per_epoch\
+ \ == 0:\n logging.info('Epoch: %s/%s', step / num_batches_per_epoch\
+ \ + 1, num_epochs)\n logging.info('Current Streaming Accuracy:\
+ \ %.4f', sess.run(accuracy))\n\n if step % 10 == 0:\n \
+ \ eval_step(sess, metrics_op=metrics_op, global_step=sv.global_step)\n\
+ \ summaries = sess.run(my_summary_op)\n \
+ \ sv.summary_computed(sess, summaries)\n\n\n else:\n \
+ \ eval_step(sess, metrics_op=metrics_op, global_step=sv.global_step)\n\
+ \n logging.info('Final Streaming Accuracy: %.4f', sess.run(accuracy))\n\
+ \n raw_images, labels, predictions = sess.run([raw_images, labels,\
+ \ predictions])\n for i in range(10):\n image, label,\
+ \ prediction = raw_images[i], labels[i], predictions[i]\n prediction_name,\
+ \ label_name = dataset.labels_to_name[prediction], dataset.labels_to_name[label]\n\
+ \ text = 'Prediction: %s \\n Ground Truth: %s' % (prediction_name,\
+ \ label_name)\n img_plot = plt.imshow(image)\n\n \
+ \ plt.title(text)\n img_plot.axes.get_yaxis().set_ticks([])\n\
+ \ img_plot.axes.get_xaxis().set_ticks([])\n plt.show()\n\
+ \n logging.info(\n 'Model evaluation has completed!\
+ \ Visit TensorBoard for more information regarding your evaluation.')\nSo after training the model and getting 94% accuracy i tried to evaluate\ + \ the model. On evaluation I get 0-1% accuracy the whole time. I investigated\ + \ this only to find that it is predicting the same class every time
\n\nlabels:\
+ \ [7, 11, 5, 1, 20, 0, 18, 1, 0, 7]\npredictions: [10, 10, 10, 10, 10, 10, 10,\
+ \ 10, 10, 10]\nCan anyone help in where i may be going wrong?
\n\ + \nEDIT:
\n\nTensorBoard accuracy and loss form training
\n\n![\"enter](\"https://i.stack.imgur.com/QdX6d.png\")
TensorBoard accuracy from evaluation
\n\nEDIT:
\n\nIve still not been able to solve this issues. I thought there\ + \ might be a problem with how I am restoring the graph in the eval script so I\ + \ tried using this to restore the model instead
\n\nsaver = tf.train.import_meta_graph('/log/model.ckpt.meta')\n\
+ \ndef restore_fn(sess):\n return saver.restore(sess, checkpoint_file)\ninstead of
\n\nvariables_to_restore = slim.get_variables_to_restore()\n\
+ \ saver = tf.train.Saver(variables_to_restore)\n\ndef restore_fn(sess):\n \
+ \ return saver.restore(sess, checkpoint_file)\nand just\
+ \ just takes a very long time to start and finally errors. I then tried using\
+ \ V1 of the writer in the saver (saver = tf.train.Saver(variables_to_restore,\
+ \ write_version=saver_pb2.SaveDef.V1)) and retrained and was unable to\
+ \ load this checkpoint at all as it said variables was missing.
I also\ + \ attempted to run my eval script with the same data it trained on just to see\ + \ if this may give different results yet I get the same.
\n\nFinally I\ + \ re-cloned the repo from the url and ran a train using the same dataset in the\ + \ tutorial and I get 0-3% accuracy when I evaluate even after getting it to 84%\ + \ whilst training. Also my checkpoints must have the correct information as when\ + \ I restart training the accuracy continues from where it left of. It feels like\ + \ i'm not doing something correctly when I restore the model. Would really appreciate\ + \ any suggestions on this as im at a dead end currently :(
\n" +- text: 'I''ve just started using tensorflow for a project I''m working on. The
+ program aims to be a binary classifier with input being 12 features. The output
+ is either normal patient or patient with a disease. The prevalence of the disease
+ is quite low and so my dataset is very imbalanced, with 502 examples of normal
+ controls and only 38 diseased patients. For this reason, I''m trying to use tf.nn.weighted_cross_entropy_with_logits
+ as my cost function.
The code is based on the iris custom estimator from the official tensorflow
+ documentation, and works with tf.losses.sparse_softmax_cross_entropy
+ as the cost function. However, when I change to weighted_cross_entropy_with_logits,
+ I get a shape error and I''m not sure how to fix this.
ValueError: logits and targets must have the same shape ((?, 2) vs
+ (?,))
+
+ I have searched and similar problems have been solved by just reshaping the
+ labels - I have tried to do this unsuccessfully (and don''t understand why tf.losses.sparse_softmax_cross_entropy
+ works fine and the weighted version does not).
My full code is here + + https://gist.github.com/revacious/83142573700c17b8d26a4a1b84b0dff7
+ + +Thanks!
+ + ' +- text: 'In the documentation it seems they focus on how to save and restore tf.keras.models, + but i was wondering how do you save and restore models trained customly through + some basic iteration loop?
+ + +Now that there isnt a graph or a session, how do we save structure defined + in a tf function that is customly built without using layer abstractions?
+ + ' +- text: "I simply have train = optimizer.minimize(loss = tf.constant(4,dtype=\"\
+ float32\")) Line of code that i change before everything is working.
Why it is giving error ? Because documentation say it can be tensor Here is Docs
\n\nW = tf.Variable([0.5],tf.float32)\n\
+ b = tf.Variable([0.1],tf.float32)\nx = tf.placeholder(tf.float32)\ny= tf.placeholder(tf.float32)\n\
+ discounted_reward = tf.placeholder(tf.float32,shape=[4,], name=\"discounted_reward\"\
+ )\nlinear_model = W*x + b\n\nsquared_delta = tf.square(linear_model - y)\nprint(squared_delta)\n\
+ loss = tf.reduce_sum(squared_delta*discounted_reward)\nprint(loss)\noptimizer\
+ \ = tf.train.GradientDescentOptimizer(0.01)\ntrain = optimizer.minimize(loss =\
+ \ tf.constant(4,dtype=\"float32\"))\ninit = tf.global_variables_initializer()\n\
+ sess = tf.Session()\n\nsess.run(init)\n\nfor i in range(3):\n sess.run(train,{x:[1,2,3,4],y:[0,-1,-2,-3],discounted_reward:[1,2,3,4]})\n\
+ \nprint(sess.run([W,b]))\n\n\n
I really need this thing\ + \ to work. In this particular example we can have other ways to solve it but i\ + \ need it to work as my actual code can do this only
\n\nError is\n\ + \n
> ValueError: No gradients provided for any variable, check your\
+ \ graph\n> for ops that do not support gradients, between variables\n> [\"\
+ <tf.Variable 'Variable:0' shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable\
+ \ 'Variable_1:0' shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_2:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_3:0' shape=(1,)\
+ \ dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_4:0' shape=(1,) dtype=float32_ref>\"\
+ ,\n> \"<tf.Variable 'Variable_5:0' shape=(1,) dtype=float32_ref>\",\n\
+ > \"<tf.Variable 'Variable_6:0' shape=(1,) dtype=float32_ref>\",\n>\
+ \ \"<tf.Variable 'Variable_7:0' shape=(1,) dtype=float32_ref>\",\n> \"\
+ <tf.Variable 'Variable_8:0' shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable\
+ \ 'Variable_9:0' shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_10:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_11:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_12:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_13:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_14:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_15:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_16:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_17:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_18:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_19:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_20:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_21:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_22:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_23:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_24:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_25:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_26:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_27:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_28:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_29:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_30:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_31:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_32:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_33:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_34:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_35:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_36:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_37:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_38:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_39:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_40:0'\
+ \ shape=(1,) dtype=float32_ref>\",\n> \"<tf.Variable 'Variable_41:0'\
+ \ shape=(1,) dtype=float32_ref>\"] and loss\n> Tensor(\"Const_4:0\", shape=(),\
+ \ dtype=float32).\nI found in the tensorflow doc:
\n\n\nstacked_lstm\
+ \ = tf.contrib.rnn.MultiRNNCell([lstm] * number_of_layers,\n ...\n\
+
I need to use MultiRNNCell
\n\nbut, I write those lines
\n\ + \n\na = [tf.nn.rnn_cell.BasicLSTMCell(10)]*3\nprint id(a[0]), id(a[1])\n\
+
Its output is [4648063696 4648063696].
Can MultiRNNCell use the same object BasicLSTMCell\
+ \ as a list for parameter?
- '
I want to train a convolutional neural network with TensorFlow to do multi-output multi-class classification.
\n\nFor example: If we take the MNIST sample set and always combine two random images two a single one and then want to classify the resulting image. The result of the classification should be the two digits shown in the image.
\n\nSo the output of the network could have the shape [-1, 2, 10] where the first dimension is the batch, the second represents the output (is it the first or the second digit) and the third is the "usual" classification of the shown digit.
\n\nI tried googling for this for a while now, but wasn\'t able find something useful. Also, I don\'t know if multi-output multi-class classification is the correct naming for this task. If not, what is the correct naming? Do you have any links/tutorials/documentations/papers explaining what I\'d need to do to build the loss function/training operations?
\n\nWhat I tried was to split up the output of the network into the single outputs with tf.split and then use softmax_cross_entropy_with_logits on every single output. The result I averaged over all outputs but it doesn\'t seem to work. Is this even a reasonable way?
\n' - '
When operating in graph mode in TF1, I believe I needed to wire up
\n\ntraining=Trueandtraining=Falsevia feeddicts when I was using the functional-style API. What is the proper way to do this in TF2?I believe this is automatically handled when using
\n\ntf.keras.Sequential. For example, I don\'t need to specifytrainingin the following example from the docs:
\n\nmodel = tf.keras.Sequential([\n tf.keras.layers.Conv2D(32, 3, activation=\'relu\',\n kernel_regularizer=tf.keras.regularizers.l2(0.02),\n input_shape=(28, 28, 1)),\n tf.keras.layers.MaxPooling2D(),\n tf.keras.layers.Flatten(),\n tf.keras.layers.Dropout(0.1),\n tf.keras.layers.Dense(64, activation=\'relu\'),\n tf.keras.layers.BatchNormalization(),\n tf.keras.layers.Dense(10, activation=\'softmax\')\n])\n\n# Model is the full model w/o custom layers\nmodel.compile(optimizer=\'adam\',\n loss=\'sparse_categorical_crossentropy\',\n metrics=[\'accuracy\'])\n\nmodel.fit(train_data, epochs=NUM_EPOCHS)\nloss, acc = model.evaluate(test_data)\nprint("Loss {:0.4f}, Accuracy {:0.4f}".format(loss, acc))\nCan I also assume that keras automagically handles this when training with the functional api? Here is the same model, rewritten using the function api:
\n\n
\n\ninputs = tf.keras.Input(shape=((28,28,1)), name="input_image")\nhid = tf.keras.layers.Conv2D(32, 3, activation=\'relu\',\n kernel_regularizer=tf.keras.regularizers.l2(0.02),\n input_shape=(28, 28, 1))(inputs)\nhid = tf.keras.layers.MaxPooling2D()(hid)\nhid = tf.keras.layers.Flatten()(hid)\nhid = tf.keras.layers.Dropout(0.1)(hid)\nhid = tf.keras.layers.Dense(64, activation=\'relu\')(hid)\nhid = tf.keras.layers.BatchNormalization()(hid)\noutputs = tf.keras.layers.Dense(10, activation=\'softmax\')(hid)\nmodel_fn = tf.keras.Model(inputs=inputs, outputs=outputs)\n\n# Model is the full model w/o custom layers\nmodel_fn.compile(optimizer=\'adam\',\n loss=\'sparse_categorical_crossentropy\',\n metrics=[\'accuracy\'])\n\nmodel_fn.fit(train_data, epochs=NUM_EPOCHS)\nloss, acc = model_fn.evaluate(test_data)\nprint("Loss {:0.4f}, Accuracy {:0.4f}".format(loss, acc))\nI\'m unsure if
\n\nhid = tf.keras.layers.BatchNormalization()(hid)needs to behid = tf.keras.layers.BatchNormalization()(hid, training)?A colab for these models can be found here.
\n' - '
I am using Tensorflow 2.0 and am able to train a CNN for image classification of 3-channel images. I perform image preprocessing within the data input pipeline (shown below) and would like to include the preprocessing functionality in the served model itself. My model is served with a TF Serving Docker container and the Predict API.
\n\nThe data input pipeline for training is based on the documentation at https://www.tensorflow.org/alpha/tutorials/load_data/images.
\n\nMy pipeline image preprocessing function is load_and_preprocess_from_path_label:
\n\n
\n\ndef load_and_preprocess_path(image_path):\n\n # Load image\n image = tf.io.read_file(image_path)\n image = tf.image.decode_png(image)\n\n # Normalize to [0,1] range\n image /= 255\n\n # Convert to HSV and Resize\n image = tf.image.rgb_to_hsv(image)\n image = tf.image.resize(image, [HEIGHT, WIDTH])\n\n return image\n\ndef load_and_preprocess_from_path_label(image_path, label):\n\n return load_and_preprocess_path(image_path), label\nWith lists of image paths, the pipeline prefetches and performs image preprocessing using tf functions within load_and_preprocess_from_path_label:
\n\n
\n\nall_image_paths, all_image_labels = parse_labeled_image_paths()\nx_train, x_test, y_train, y_test = sklearn.model_selection.train_test_split(all_image_paths, all_image_labels, test_size=0.2)\n\n# Create a TensorFlow Dataset of training images and labels\nds = tf.data.Dataset.from_tensor_slices((x_train, y_train))\nimage_label_ds = ds.map(load_and_preprocess_from_path_label)\n\nBATCH_SIZE = 32\nIMAGE_COUNT = len(all_image_paths)\n\nds = image_label_ds.apply(tf.data.experimental.shuffle_and_repeat(buffer_size=IMAGE_COUNT))\nds = ds.batch(BATCH_SIZE)\nds = ds.prefetch(buffer_size=AUTOTUNE)\n\n# Create image pipeline for model\nimage_batch, label_batch = next(iter(ds))\nfeature_map_batch = model(image_batch)\n\n# Train model\nmodel.fit(ds, epochs=5)\nPrevious Tensorflow examples I\'ve found use serving_input_fn(), and utilized tf.placeholder which seems to no longer exist in Tensorflow 2.0.
\n\nAn example for serving_input_fn in Tensorflow 2.0 is shown on https://www.tensorflow.org/alpha/guide/saved_model. Since I am using the Predict API, it looks like I would need something similar to:
\n\n
\n\nserving_input_fn = tf.estimator.export.build_raw_serving_input_receiver_fn(...)\n\n# Save the model with the serving preprocessing function\nmodel.export_saved_model(MODEL_PATH, serving_input_fn)\n\nIdeally, the served model would accept a 4D Tensor of 3-channel image samples of any size and would perform the initial image preprocessing on them (decode image, normalize, convert to HSV, and resize) before classifying.
\n\nHow can I create a serving_input_fn in Tensorflow 2.0 with a preprocessing function similar to my load_and_preprocess_path function?
\n'
- '
I ma trying to understand tf.rank function in tensorflow. From the documentation here, I understood that rank should return the number of distinct elements in the tensor.
\n\nHere x and weights are 2 distinct 2*2 tensors with 4 distinct elemnts in each of them. However, rank() function outputs are:
\n\n\n
\n\nTensor("Rank:0", shape=(), dtype=int32) Tensor("Rank_1:0", shape=(),\n dtype=int32)
\nAlso, for the tensor x, I used tf.constant() with dtype = float to convert ndarray into float32 tensor but the rank() still outputs as int32.
\n\n
\n\ng = tf.Graph()\nwith g.as_default():\n weights = tf.Variable(tf.truncated_normal([2,2]))\n x = np.asarray([[1 , 2], [3 , 4]])\n x = tf.constant(x, dtype = tf.float32)\n y = tf.matmul(weights, x)\n print (tf.rank(x), tf.rank(weights))\n\n\nwith tf.Session(graph = g) as s:\n tf.initialize_all_variables().run()\n print (s.run(weights), s.run(x))\n print (s.run(y))\nHow should I interpret the output.
\n' - "
My weights are defined as
\n\n
\n\nweights = {\n 'W_conv1': tf.get_variable('W_conv1', shape=[...], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.01)),\n 'W_conv2': tf.get_variable('W_conv2', shape=[...], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.01)),\n 'W_conv3': tf.get_variable('W_conv3', shape=[...], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.01)),\n ...\n}\n\n# conv2d network\n...\nI want to use the weights decay so I add, for example, the argument
\n\n
\n\nregularizer=tf.contrib.layers.l1_regularizer(0.0005)\nto the
\n\ntf.get_variable. Now I'm wondering if during the evaluation phase this is still correct or maybe I have to set the regularizer factor to 0.There is also another argument
\n\ntrainable. The documentation saysIf True also add the variable to the graph collection GraphKeys.TRAINABLE_VARIABLES.which is not clear to me. Should I use it?Can someone explain to me if the weights decay effects in a sort of wrong way the evaluation step? How can I solve in that case?
\n" - "
By this piece of code from the documentation, we can create multiple features to feed batches of data into a DNN model:
\n\n
\n\nmy_feature_columns = []\nfor key in train_x.keys():\n my_feature_columns.append(tf.feature_column.numeric_column(key=key))\nBut the problem is what is the proper way to transform the original features before they are fed to the input layer? Typical transformations that I can think of include normalization and clipping.
\n\n
\n\ntf.feature_column.numeric_columndoes have a parameter specifying the normalization function. But the example in the doc only demonstrate a scenario where the normalization factors are pre-defined and fixed, likelambda x: (x-3.2)/1.5. How can I perform normalization (e.g.MinMaxScalerin sklearn) across all those features without knowing its maximum and minimum beforehand.Also, is there any pipeline implementation where it's possible to do all sorts of feature transformations before they go into the input layer? Is creating a custom estimator
\n"tf.estimator.Estimatorthe answer to this problem? or anything else I'm not aware of.
- '
I\'m looking to use Tensorflow to train a neural network model for classification, and I want to read data from a CSV file, such as the Iris data set.
\n\nThe Tensorflow documentation shows an example of loading the Iris data and building a prediction model, but the example uses the high-level
\n'tf.contrib.learnAPI. I want to use the low-level Tensorflow API and run gradient descent myself. How would I do that? - '
In the following code, I want dense matrix
\n\nBto left multiply a sparse matrixA, but I got errors.
\n\nimport tensorflow as tf\nimport numpy as np\n\nA = tf.sparse_placeholder(tf.float32)\nB = tf.placeholder(tf.float32, shape=(5,5))\nC = tf.matmul(B,A,a_is_sparse=False,b_is_sparse=True)\nsess = tf.InteractiveSession()\nindices = np.array([[3, 2], [1, 2]], dtype=np.int64)\nvalues = np.array([1.0, 2.0], dtype=np.float32)\nshape = np.array([5,5], dtype=np.int64)\nSparse_A = tf.SparseTensorValue(indices, values, shape)\nRandB = np.ones((5, 5))\nprint sess.run(C, feed_dict={A: Sparse_A, B: RandB})\nThe error message is as follows:
\n\n
\n\nTypeError: Failed to convert object of type <class \'tensorflow.python.framework.sparse_tensor.SparseTensor\'> \nto Tensor. Contents: SparseTensor(indices=Tensor("Placeholder_4:0", shape=(?, ?), dtype=int64), values=Tensor("Placeholder_3:0", shape=(?,), dtype=float32), dense_shape=Tensor("Placeholder_2:0", shape=(?,), dtype=int64)). \nConsider casting elements to a supported type.\nWhat\'s wrong with my code?
\n\nI\'m doing this following the documentation and it says we should use
\n\na_is_sparseto denote whether the first matrix is sparse, and similarly withb_is_sparse. Why is my code wrong?As is suggested by vijay, I should use
\n\nC = tf.matmul(B,tf.sparse_tensor_to_dense(A),a_is_sparse=False,b_is_sparse=True)I tried this but I met with another error saying:
\n\n
\n\nCaused by op u\'SparseToDense\', defined at:\n File "a.py", line 19, in <module>\n C = tf.matmul(B,tf.sparse_tensor_to_dense(A),a_is_sparse=False,b_is_sparse=True)\n File "/home/fengchao.pfc/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/sparse_ops.py", line 845, in sparse_tensor_to_dense\n name=name)\n File "/home/mypath/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/sparse_ops.py", line 710, in sparse_to_dense\n name=name)\n File "/home/mypath/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/gen_sparse_ops.py", line 1094, in _sparse_to_dense\n validate_indices=validate_indices, name=name)\n File "/home/mypath/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op\n op_def=op_def)\n File "/home/mypath/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2506, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File "/home/mypath/anaconda2/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1269, in __init__\n self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): indices[1] = [1,2] is out of order\n[[Node: SparseToDense = SparseToDense[T=DT_FLOAT, Tindices=DT_INT64, validate_indices=true, _device="/job:localhost/replica:0/task:0/cpu:0"](_arg_Placeholder_4_0_2, _arg_Placeholder_2_0_0, _arg_Placeholder_3_0_1, SparseToDense/default_value)]]\nThank you all for helping me!
\n' - "
I am using
\n\ntf.estimator.train_and_evaluateandtf.data.Datasetto feed data to the estimator:Input Data function:
\n\n
\n\ndef data_fn(data_dict, batch_size, mode, num_epochs=10):\n dataset = {}\n if mode == tf.estimator.ModeKeys.TRAIN:\n dataset = tf.data.Dataset.from_tensor_slices(data_dict['train_data'].astype(np.float32))\n dataset = dataset.cache()\n dataset = dataset.shuffle(buffer_size= batch_size * 10).repeat(num_epochs).batch(batch_size)\n else:\n dataset = tf.data.Dataset.from_tensor_slices(data_dict['valid_data'].astype(np.float32))\n dataset = dataset.cache()\n dataset = dataset.batch(batch_size)\n\n iterator = dataset.make_one_shot_iterator()\n next_element = iterator.get_next()\n\n return next_element\nTrain Function:
\n\n
\n\ndef train_model(data):\n tf.logging.set_verbosity(tf.logging.INFO)\n config = tf.ConfigProto(allow_soft_placement=True,\n log_device_placement=False)\n config.gpu_options.allow_growth = True\n run_config = tf.contrib.learn.RunConfig(\n save_checkpoints_steps=10,\n keep_checkpoint_max=10,\n session_config=config\n )\n\n train_input = lambda: data_fn(data, 100, tf.estimator.ModeKeys.TRAIN, num_epochs=1)\n eval_input = lambda: data_fn(data, 1000, tf.estimator.ModeKeys.EVAL)\n estimator = tf.estimator.Estimator(model_fn=model_fn, params=hps, config=run_config)\n train_spec = tf.estimator.TrainSpec(train_input, max_steps=100)\n eval_spec = tf.estimator.EvalSpec(eval_input,\n steps=None,\n throttle_secs = 30)\n\n tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)\nThe training goes fine, but when it comes to evaluation I get this error:
\n\n
\n\nOutOfRangeError (see above for traceback): End of sequence \nIf I don't use
\n\nDataset.batchon evaluation dataset (by omitting the linedataset[name] = dataset[name].batch(batch_size)indata_fn) I get the same error but after a much longer time.I can only avoid this error if I don't batch the data and use
\n\nsteps=1for evaluation, but does that perform the evaluation on the whole dataset?I don't understand what causes this error as the documentation suggests I should be able to evaluate on batches too.
\n\nNote: I get the same error when using
\n"tf.estimator.evaluateon data batches.
- '
I\'m working on a project where I have trained a series of binary classifiers with Keras, with Tensorflow as the backend engine. The input data I have is a series of images, where each binary classifier must make the prediction on the images, later I save the predictions on a CSV file.
\nThe problem I have is when I get the predictions from the first series of binary classifiers there isn\'t any warning, but when the 5th or 6th binary classifier calls the method predict on the input data I get the following warning:
\n\n
\nWARNING:tensorflow:5 out of the last 5 calls to <function\nModel.make_predict_function..predict_function at\n0x2b280ff5c158> triggered tf.function retracing. Tracing is expensive\nand the excessive number of tracings could be due to (1) creating\n@tf.function repeatedly in a loop, (2) passing tensors with different\nshapes, (3) passing Python objects instead of tensors. For (1), please\ndefine your @tf.function outside of the loop. For (2), @tf.function\nhas experimental_relax_shapes=True option that relaxes argument shapes\nthat can avoid unnecessary retracing. For (3), please refer to\nhttps://www.tensorflow.org/tutorials/customization/performance#python_or_tensor_args\nand https://www.tensorflow.org/api_docs/python/tf/function for more\ndetails.
\nTo answer each point in the parenthesis, here are my answers:
\n- \n
- The predict method is called inside a for loop. \n
- I don\'t pass tensors but a list of NumPy arrays of gray scale images, all of them with the same size in width and height. The only thing that can change is the batch size because the list can have only 1 image or more than one. \n
- As I wrote in point 2, I pass a list of NumPy arrays. \n
I have debugged my program and found that this warning always happens when the method predict is called. To summarize the code I have written is the following:
\n
\nimport cv2 as cv\nimport tensorflow as tf\nfrom tensorflow.keras.models import load_model\n# Load the models\nbinary_classifiers = [load_model(path) for path in path2models]\n# Get the images\nimages = [#Load the images with OpenCV]\n# Apply the resizing and reshapes on the images.\nmy_list = list()\nfor image in images:\n image_reworked = # Apply the resizing and reshaping on images\n my_list.append(image_reworked)\n\n# Get the prediction from each model\n# This is where I get the warning\npredictions = [model.predict(x=my_list,verbose=0) for model in binary_classifiers]\nWhat I have tried
\nI have defined a function as tf.function and putted the code of the predictions inside the tf.function like this
\n
\n@tf.function\ndef testing(models, faces):\n return [model.predict(x=faces,verbose=0) for model in models]\nBut I ended up getting the following error:
\n\n
\nRuntimeError: Detected a call to
\nModel.predictinside a\ntf.function. Model.predict is a high-level endpoint that manages\nits owntf.function. Please move the call toModel.predictoutside\nof all enclosingtf.functions. Note that you can call aModel\ndirectly on Tensors inside atf.functionlike:model(x).So calling the method
\npredictis basically already a tf.function. So it\'s useless to define a tf.function when the warning I get it\'s from that method.I have also checked those other two questions:
\n- \n
- Tensorflow 2: Getting "WARNING:tensorflow:9 out of the last 9 calls to triggered tf.function retracing. Tracing is expensive" \n
- Loading multiple saved tensorflow/keras models for prediction \n
But neither of the two questions answers my question about how to avoid this warning. Plus I have also checked the links in the warning message but I couldn\'t solve my problem.
\nWhat I want
\nI simply want to avoid this warning. While I\'m still getting the predictions from the models I noticed that the python program takes way too much time on doing predictions for a list of images.
\nWhat I\'m using
\n- \n
- Python 3.6.13 \n
- Tensorflow 2.3.0 \n
Solution
\nAfter some tries to suppress the warning from the
\npredictmethod, I have checked the documentation of Tensorflow and in one of the first tutorials on how to use Tensorflow it is explained that, by default, Tensorflow is executed in eager mode, which is useful for testing and debugging the network models. Since I have already tested my models many times, it was only required to disable the eager mode by writing this single python line of code:
\ntf.compat.v1.disable_eager_execution()Now the warning doesn\'t show up anymore.
\n' - '
I try to export a Tensorflow model but I can not find the best way to add the exogenous feature to the
\n\ntf.contrib.timeseries.StructuralEnsembleRegressor.build_raw_serving_input_receiver_fn.I use the sample from the Tensorflow contrib: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/timeseries/examples/known_anomaly.py and I just try to save the model.
\n\n
\n\n# this is the exogenous column \nstring_feature = tf.contrib.layers.sparse_column_with_keys(\n column_name="is_changepoint", keys=["no", "yes"])\n\none_hot_feature = tf.contrib.layers.one_hot_column(\n sparse_id_column=string_feature)\n\nestimator = tf.contrib.timeseries.StructuralEnsembleRegressor(\n periodicities=12, \n cycle_num_latent_values=3,\n num_features=1,\n exogenous_feature_columns=[one_hot_feature],\n exogenous_update_condition=\n lambda times, features: tf.equal(features["is_changepoint"], "yes"))\n\nreader = tf.contrib.timeseries.CSVReader(\n csv_file_name,\n\n column_names=(tf.contrib.timeseries.TrainEvalFeatures.TIMES,\n tf.contrib.timeseries.TrainEvalFeatures.VALUES,\n "is_changepoint"),\n\n column_dtypes=(tf.int64, tf.float32, tf.string),\n\n skip_header_lines=1)\n\ntrain_input_fn = tf.contrib.timeseries.RandomWindowInputFn(reader, batch_size=4, window_size=64)\nestimator.train(input_fn=train_input_fn, steps=train_steps)\nevaluation_input_fn = tf.contrib.timeseries.WholeDatasetInputFn(reader)\nevaluation = estimator.evaluate(input_fn=evaluation_input_fn, steps=1)\n\nexport_directory = tempfile.mkdtemp()\n\n###################################################### \n# the exogenous column must be provided to the build_raw_serving_input_receiver_fn. \n# But How ?\n######################################################\n\ninput_receiver_fn = estimator.build_raw_serving_input_receiver_fn()\n# -> error missing \'is_changepoint\' key \n\n#input_receiver_fn = estimator.build_raw_serving_input_receiver_fn({\'is_changepoint\' : string_feature}) \n# -> cast exception\n\nexport_location = estimator.export_savedmodel(export_directory, input_receiver_fn)\nAccording to the documentation, build_raw_serving_input_receiver_fn exogenous_features parameter : A dictionary mapping feature keys to exogenous features (either Numpy arrays or Tensors). Used to determine the shapes of placeholders for these features.
\n\nSo what is the best way to transform the one_hot_column or sparse_column_with_keys to a Tensor object ?
\n' - "
I am currently working on an optical flow project and I come across a strange error.
\n\nI have uint16 images stored in bytes in my TFrecords. When I read the TFrecords from my local machine it is giving me uint16 values, but when I deploy the same code and read it from the docker I am getting uint8 values eventhough my dtype is uint16. I mean the uint16 values are getting reduced to uint8 like 32768 --> 128.
\n\nWhat is causing this error?
\n\nMy local machine has: Tensorflow 1.10.1 and python 3.6\nMy Docker Image has: Tensorflow 1.12.0 and python 3.5
\n\nI am working on tensorflow object detection API\nWhile creating the TF records I use:
\n\n
\n\nwith tf.gfile.GFile(flows, 'rb') as fid:\n flow_images = fid.read()\nWhile reading it back I am using: tf.image.decoderaw
\n\nDataset: KITTI FLOW 2015
\n"
New to TensorFlow, so apologies for newbie question.
+preds = model("In the documentation it seems they focus on how to save and restore tf.keras.models, but i was wondering how do you save and restore models trained customly through some basic iteration loop?
-Following this tutorial but instead of using image data I am using numerical data.
- -Load the dataset:
- -train_dataset_url = \"xxx.csv\"
-train_dataset_fp = tf.keras.utils.get_file(
- fname=os.path.basename(train_dataset_url),
- origin=train_dataset_url)
-Make training dataset:
- -batch_size = 32
-
-train_dataset = tf.contrib.data.make_csv_dataset(
- train_dataset_fp,
- batch_size,
- column_names=column_names,
- label_name=label_name,
- num_epochs=1)
-Train classified model using:
- -model = tf.keras.Sequential([
- tf.keras.layers.Dense(10, activation=tf.nn.relu, input_shape=(1,)),
- tf.keras.layers.Dense(10, activation=tf.nn.relu),
- tf.keras.layers.Dense(4)
-])
But when I \"test\" the model with the same inputs:
- -predictions = model(features)
I receive the error:
- -InvalidArgumentError: cannot compute MatMul as input #0(zero-based) was expected to be a float tensor but is a int32 tensor [Op:MatMul]
It's possible I have missed something fundamental. I feel like I need to specify a type somewhere.
+Now that there isnt a graph or a session, how do we save structure defined in a tf function that is customly built without using layer abstractions?
") ``` @@ -399,21 +457,20 @@ train_dataset = tf.contrib.data.make_csv_dataset( ## Training Details ### Training Set Metrics -| Training set | Min | Median | Max | -|:-------------|:----|:--------|:-----| -| Word count | 15 | 336.765 | 3755 | +| Training set | Min | Median | Max | +|:-------------|:----|:---------|:-----| +| Word count | 15 | 330.0667 | 3755 | | Label | Training Sample Count | |:------|:----------------------| -| 0 | 500 | -| 1 | 500 | +| 0 | 450 | +| 1 | 450 | ### Training Hyperparameters -- batch_size: (16, 16) -- num_epochs: (2, 2) +- batch_size: (16, 2) +- num_epochs: (1, 16) - max_steps: -1 -- sampling_strategy: oversampling -- num_iterations: 20 +- sampling_strategy: unique - body_learning_rate: (2e-05, 1e-05) - head_learning_rate: 0.01 - loss: CosineSimilarityLoss @@ -428,11 +485,10 @@ train_dataset = tf.contrib.data.make_csv_dataset( - load_best_model_at_end: True ### Training Results -| Epoch | Step | Training Loss | Validation Loss | -|:-------:|:--------:|:-------------:|:---------------:| -| 0.0004 | 1 | 0.3216 | - | -| **1.0** | **2500** | **0.0001** | **0.3943** | -| 2.0 | 5000 | 0.0 | 0.3993 | +| Epoch | Step | Training Loss | Validation Loss | +|:-------:|:---------:|:-------------:|:---------------:| +| 0.0000 | 1 | 0.2951 | - | +| **1.0** | **25341** | **0.0** | **0.2473** | * The bold row denotes the saved checkpoint. ### Framework Versions