Refactor.

_.py

@@ -1,14 +0,0 @@
-import os
-
-start_image = 12000
-
-from tensorflow.keras.models import load_model
-model = load_model("saved_model/1")
-
-from data_utils import nload, ims, DataSetFromFolder
-data = nload("/home/jkwiatkowski/all/dataset/arr/val.npy")
-indexes = nload("/home/jkwiatkowski/all/dataset/arr/val_target.npy")
-
-folders = DataSetFromFolder("/home/jkwiatkowski/all/dataset/arr/RAVEN-10000-release/RAVEN-10000", file_type="dir")
-properties = DataSetFromFolder(folders[:], file_type="xml", extension="val")
-

models.py

@@ -1,6 +1,5 @@
 import tensorflow as tf
 from config_utils import tf_gpu
-from data_utils import DataSetFromFolder
 
 tf_gpu()
 tf.experimental.numpy.experimental_enable_numpy_behavior(prefer_float32=True)
@@ -11,19 +10,7 @@ from datasets import load_dataset
 repo = "jkwiatkowski/raven"
 
 data = load_dataset(repo, split="val")
+properties = load_dataset(repo + "_properties", split="val")
 model = from_pretrained_keras(repo)
 
-properties =  load_dataset(repo + "_properties", split="val")
 START_IMAGE = 12000
-
-
-# def convert(data):
-#     return {
-#         'inputs': tf.cast(data['inputs'], dtype="uint8"),
-#         'index': tf.cast(data['index'], dtype="uint8")[..., None],
-#         'target': tf.cast(data['target'], dtype="int8"),
-#     }
-#
-# model(convert(data[0:1]))
-
-print("xD")

raven_utils/data.py

@@ -1,46 +0,0 @@
-import os
-
-import tensorflow as tf
-
-from models_utils import INPUTS, TARGET
-
-from raven_utils.config.constant import RAVEN, LABELS, INDEX, FEATURES, RAV_METRICS, IMP_RAV_METRICS, ACC_NO_GROUP
-
-
-from typing import Any
-
-from data_utils import pre, Data, gather, vec, resize
-from data_utils.data_generator import DataGenerator
-from funcy import identity
-
-
-def get_data(data, batch_size, steps=None, val_steps=None):
-    if val_steps is None:
-        val_steps = steps
-    fn = identity
-    train_target_index = data[4] + 8
-    train_generator = DataGenerator({
-        INPUTS: Data(data[0], fn),
-        TARGET: Data(data[2], identity),
-        LABELS: Data(data[2], identity),
-        INDEX: train_target_index[:, None],
-        # FEATURES: data[6]
-    },
-        batch=batch_size,
-        steps=steps
-    )
-    val_target_index = data[5] + 8
-    val_data = {
-        INPUTS: Data(data[1], fn),
-        TARGET: Data(data[3], identity),
-        LABELS: Data(data[3], identity),
-        INDEX: val_target_index[:, None],
-        # FEATURES: data[7]
-    }
-    val_generator = DataGenerator(
-        val_data,
-        batch=batch_size,
-        sampler="val",
-        steps=val_steps
-    )
-    return train_generator, val_generator

raven_utils/decode.py

@@ -1,9 +1,24 @@
 import numpy as np
-from data_utils import np_split
-from ml_utils import lw
-from models_utils.ops import ibin
-
 import raven_utils as rv
+import tensorflow as tf
+
+
+def np_split(ary, indices_or_sections, axis=-1):
+    return np.split(ary, np.cumsum(indices_or_sections), axis)[:-1]
+
+def lw(data, none_empty=True, convert_tuple=True):
+    if isinstance(data, list):
+        return data
+    elif isinstance(data, tuple) and convert_tuple:
+        return list(data)
+    if none_empty and data is None:
+        return []
+    return [data]
+
+
+
+def ibin(x):
+    return tf.cast(bin(x), dtype=tf.int32)
 
 
 def output(x, split_fn=np_split, predict_fn_1=np.argmax, predict_fn_2=ibin):
@@ -62,8 +77,6 @@ def decode_target(target):
     return target_group, target_slot, target_properties_splited
 
 
-
-
 def decode_target_flat(target):
     t = decode_target(target)
     return t[0], t[1], t[2][0], t[2][1], t[2][2]
@@ -78,17 +91,19 @@ def demask(target, mask=None, group=None, zeroes=None):
         mask = RangeMask()(group).numpy()
     if zeroes is None:
         return np.concatenate([t[mask] for t in lw(target[1:])])
-    return np.concatenate([target[0][None]] + [t * mask for t in lw(target[1:])],axis=-1)
+    return np.concatenate([target[0][None]] + [t * mask for t in lw(target[1:])], axis=-1)
 
 
-def target_mask(mask,right=1):
+def target_mask(mask, right=1):
     shape = mask.shape
-    return np.concatenate([np.ones([shape[0], 1]) ,mask, np.repeat(mask,3,axis=1), np.ones([shape[0], right])],axis=1)
+    return np.concatenate([np.ones([shape[0], 1]), mask, np.repeat(mask, 3, axis=1), np.ones([shape[0], right])],
+                          axis=1)
 
 
 def get_full_range_mask(mask):
     return np.concatenate([mask, np.repeat(mask, 3, axis=-1)], axis=-1)
 
+
 def compare(target, predict, mask):
     target_comp = target[:, 1:rv.target.END_INDEX]
     predict_comp = predict[:, 1:rv.target.END_INDEX]

raven_utils/draw.py

@@ -1,19 +1,16 @@
 import numpy as np
-from data_utils import take, EXIST, COR
-from data_utils.image import draw_images, add_text
 from funcy import identity
-from ml_utils import none, filter_keys, lu
-from models_utils import is_model
-from models_utils import ops as K
-from tensorflow.python.saved_model.load import Loader
 
 from raven_utils.constant import PROPERTY, TARGET, INPUTS
 from raven_utils.decode import decode_target, target_mask
+from raven_utils.image import EXIST, COR, draw_images, add_text
 from raven_utils.render.rendering import render_panels
 from raven_utils.render_ import TYPES, SIZES
+from raven_utils.tools import none, is_model, filter_keys
 from raven_utils.uitls import get_val_index
 
 
+
 def draw_board(images, target=None, predict=None, image=None, desc=None, layout=None, break_=20):
     if image != "target" and predict is not None:
         image = images[predict:predict + 1]
@@ -78,9 +75,6 @@ def draw_raven(data, predict=None, pre_fn=identity, layout=1):
         res = predict(d)
         pro = np.array(target_mask(res['mask'].numpy()) * res["predict"].numpy(),dtype=np.int8)
         predict = pre_fn(render_panels(pro, target=False)[None])[0]
-        # from data_utils import ims
-        # ims(1 - predict[0])
-    # if target is not None:
     target = data[TARGET]
     target_index = data["index"]
     images = data[INPUTS]

raven_utils/group.py

@@ -1,5 +1,3 @@
-import numpy as np
-
 NAMES = ['center_single',
          'distribute_four',
          'distribute_nine',

raven_utils/image.py

@@ -0,0 +1,492 @@
+import math
+import os
+from itertools import groupby
+import numpy as np
+from matplotlib import pyplot as plt, colors
+from skimage.transform import resize
+
+# todo different backend maybe diffrent files thata have partial
+# todo depend on mode different min and max
+from raven_utils.tools import is_num, il, image_type, lw, is_int, dict_from_list, if_images, download_file, \
+    dir_decorator, list_dir
+
+TRAIN = "train"
+VAL = "val"
+TEST = "test"
+COLOR = "color"
+SIZE = "size"
+EXIST = "exist"
+COR = "cor"
+TARGET_SIZE = "target_size"
+
+
+# https://github.com/qubvel/efficientnet/blob/master/efficientnet/preprocessing.py
+MAP_INTERPOLATION_TO_ORDER = {
+    "nearest": 0,
+    "bilinear": 1,
+    "biquadratic": 2,
+    "bicubic": 3,
+}
+
+
+def center_crop_and_resize(image, image_size, crop_padding=32, interpolation="bicubic"):
+    assert image.ndim in {2, 3}
+    assert interpolation in MAP_INTERPOLATION_TO_ORDER.keys()
+
+    h, w = image.shape[:2]
+
+    padded_center_crop_size = int(
+        (image_size / (image_size + crop_padding)) * min(h, w)
+    )
+    offset_height = ((h - padded_center_crop_size) + 1) // 2
+    offset_width = ((w - padded_center_crop_size) + 1) // 2
+
+    image_crop = image[
+                 offset_height: padded_center_crop_size + offset_height,
+                 offset_width: padded_center_crop_size + offset_width,
+                 ]
+    resized_image = resize(
+        image_crop,
+        (image_size, image_size),
+        order=MAP_INTERPOLATION_TO_ORDER[interpolation],
+        preserve_range=True,
+    )
+
+    return resized_image
+
+
+def im_conv(image):
+    return image.astype(np.float32) / 255.
+
+
+def add_dim(x):
+    return x[..., None]
+
+
+def im_conv_black(image):
+    return image[..., np.newaxis].astype(np.float32) / 255.
+
+
+def im_resolve(image):
+    return (image * 255).astype(np.uint8)
+
+
+def minmax(data, axis=None, scale=1.0):
+    return ((data - np.min(data, axis=axis)) / (np.max(data, axis=axis) - np.min(data, axis=axis))) * scale
+
+
+def inverse(x):
+    return 1.0 - x
+
+
+def standarize(img, scaler=0.15, batch=True):
+    indexes = tuple(range(1 if batch else 0, img.ndim))
+    img = np.array(img)
+    img -= np.mean(img, axis=indexes)[tuple([slice(None)] + [None] * len(indexes))]
+    img /= np.std(img, axis=indexes)[tuple([slice(None)] + [None] * len(indexes))] + 1e-5
+    img *= scaler
+    return img
+
+
+def image_standardization(img, scaler=0.15):
+    indexes = tuple(range(1, 4))
+    img -= np.mean(img, axis=indexes)
+    img /= np.std(img, axis=indexes) + 1e-5
+    img *= scaler
+    return img
+
+
+def clip_standardized(img, min=0, max=1):
+    img += (max - min) / 2
+    return np.clip(img, min, max)
+
+
+def deprocess_image(img, clip=True):
+    # Normalize array: center on 0., ensure variance is 0.15
+    img = image_standardization(img)
+    if clip:
+        img = clip_standardized(img)
+    return img
+
+
+# def im_conv_tf(image,dtype=tf.float32):
+#     return tf.image.convert_image_dtype(image,dtype)
+
+# todo Grid on edges
+# todo different channels
+def draw_images(data, col=-1, row=-1, grid=True, border=None, mark=None, grid_args={}, border_kwargs={}, *args,
+                **kwargs):
+    figure, shape = create_grid(data, row=row, col=col, *args, **kwargs)
+    if border is not None:
+        add_border(figure, border, **border_kwargs)
+    figure = create_image_from_grid(figure, *args, **kwargs)
+
+    if grid:
+        figure = add_grid(figure, shape[-3:], **grid_args)
+    return figure
+
+
+def draw_images2(data, col=-1, row=-1, grid=True, color=1, border=None, border_kwargs={}, *args,
+                 **kwargs):
+    figure, shape = create_grid(data, row=row, col=col, *args, **kwargs)
+    if border is not None:
+        add_border(figure, border, **border_kwargs)
+    figure = fill_canvas(figure, color=color, grid=grid)
+    return figure
+
+
+def draw_images3(data, col=-1, row=-1, grid=True, border=None, mark=None, grid_args={}, border_kwargs={}, *args,
+                 **kwargs):
+    if len(np.asarray(data).shape) < 4:
+        return data
+
+    return draw_images(data=data,
+                       col=col,
+                       row=row,
+                       grid=grid,
+                       border=border,
+                       mark=mark,
+                       grid_args=grid_args,
+                       border_kwargs=border_kwargs,
+                       *args,
+                       **kwargs
+                       )
+
+
+def fill_canvas(a, color=1, grid=10):
+    if not isinstance(a, np.ndarray):
+        a = np.asarray(a)
+    if grid is True:
+        grid = 10
+    shape = a.shape
+    move_r = shape[2] + grid
+    move_c = shape[3] + grid
+    if color is None:
+        color = 0
+    if is_num(color):
+        color = np.full((move_r * shape[0], move_c * shape[1]) + shape[-1:], fill_value=color, dtype=a.dtype)
+    for r in range(shape[0]):
+        for c in range(shape[1]):
+            color[move_r * r:move_r * r + shape[2], move_c * c:move_c * c + shape[3]] = a[r, c]
+    return color
+
+
+def create_grid(
+        data,
+        row=-1,
+        col=-1,
+        mode="auto",
+        # Method for creating grid if row and col not set. Even evenly for row and cows. Shape take shape from input data. Auto use shape for input data that have more then 4 dimention, otherwise even.
+        swap=False,  # swaping row and columns
+        *args, **kwargs):
+    if il(data):
+        for i, d in enumerate(data):
+            if len(d.shape) > 4:
+                data[i] = draw_images(d)
+            elif len(d.shape) < 3:
+                data[i] = d[..., np.newaxis]
+        shapes = [d.shape for d in data]
+        # padding to make all images same size
+        if not all_equal(shapes):
+            mh = np.max([sh[-3] for sh in shapes])
+            mw = np.max([sh[-2] for sh in shapes])
+            canvas = np.zeros((len(data), mh, mw, shapes[0][-1]))
+            for j, d in enumerate(data):
+                canvas[j, :d.shape[-3], :d.shape[-2]] = d
+            data = canvas
+    data = np.asarray(data)
+    shape = data.shape
+    # check in case of no channels
+    if shape[-1] > 4:
+        data = data[..., np.newaxis]
+        shape = data.shape
+    if swap and len(shape) > 4:
+        data = data.swapaxes(-4, -5)
+    h = shape[-3]
+    w = shape[-2]
+    c = shape[-1]
+    if len(shape) > 4:
+        im_no = shape[-5] * shape[-4]
+    elif len(shape) == 4:
+        im_no = shape[-4]
+    else:
+        return data, shape
+    if row == -1 and col == -1:
+        if mode == "auto":
+            mode = "shape" if len(shape) > 4 else "even"
+        if mode == "shape":
+            row = shape[-5] if len(shape) > 4 else shape[-4]
+        elif mode == "even":
+            row = math.ceil(np.sqrt(im_no))
+        else:
+            row = shape[-5] if len(shape) > 4 else math.ceil(np.sqrt(im_no))
+    # eh = -1 if row == -1 else row * h
+    # ew = -1 if column == -1 else column * w
+    # if number of images is less then available slots in grid add blank images
+    if row == -1:
+        eh = -1
+        m = im_no % col
+        if m != 0:
+            data = np.concatenate([data, np.zeros((col - m,) + shape[-3:])], axis=-4)
+    else:
+        eh = row * h
+    if col == -1:
+        ew = -1
+        m = im_no % row
+        if m != 0:
+            data = np.concatenate([data, np.zeros((row - m,) + shape[-3:])], axis=-4)
+    else:
+        ew = col * h
+    figure = data.reshape(row, col, h, w, c)
+    import time
+    time.sleep(0.1)
+    return figure, shape
+
+
+def create_image_from_grid(grid, *args, **kwargs):
+    shape = grid.shape
+    figure = grid.swapaxes(1, 2).reshape(shape[0] * shape[2], shape[1] * shape[3], shape[4])
+    return figure
+
+
+# todo For more dimention
+def add_grid(figure, shape, left=True, right=True, color=1.00, hor=True, ver=True):
+    revers = False
+    if len(figure.shape) == 2:
+        figure = figure[..., None]
+        reverse = True
+    if hor:
+        figure[shape[0]::shape[0], :, :] = color
+    if ver:
+        figure[:, shape[1]::shape[1], :] = color
+    if left:
+        figure[0, :, :] = color
+        figure[0, :] = color
+    if right:
+        figure[figure.shape[0] - 1, :, :] = color
+        figure[:, ::figure.shape[1] - 1, :] = color
+    if revers:
+        figure = figure[..., 0]
+    return figure
+
+
+def add_border(figure, cor, exist=True, size=4, color="auto"):
+    if color == "auto":
+        info = image_type(figure)
+        if info.mode == "RGB":
+            color = np.array(colors.to_rgb("red")) * info.range_[1]
+        else:
+            color = info.range_[1]
+        if info.reverse:
+            color = info.range_[1] - color
+
+    shape = figure.shape
+    for c in lw(cor):
+        if not isinstance(c, dict):
+            c = dict_from_list([COR, EXIST, SIZE, COLOR], c)
+        c = {
+            **{
+                EXIST: exist if il(exist) else [exist] * 4,
+                SIZE: size,
+                COLOR: color
+            },
+            **c
+        }
+
+        if is_int(lw(c[EXIST])[0]):
+            c[EXIST] = K.mask(4, c[EXIST])
+        if is_int(c[COR]):
+            c[COR] = c[COR] // shape[1], c[COR] % shape[1]
+            isinstance(lw(c[EXIST])[0], np.integer)
+
+        cr = c[COR]
+        if c[EXIST][0]:
+            figure[cr[0], cr[1], 0:c[SIZE], :] = c[COLOR]
+        if c[EXIST][1]:
+            figure[cr[0], cr[1], :, shape[3] - c[SIZE]:shape[3]] = c[COLOR]
+        if c[EXIST][2]:
+            figure[cr[0], cr[1], shape[2] - c[SIZE]:shape[2], :] = c[COLOR]
+        if c[EXIST][3]:
+            figure[cr[0], cr[1], :, 0:c[SIZE]] = c[COLOR]
+    return figure
+
+
+# todo hgih tmp fix
+def get_digitilizer(min=0, max=1, steps=10, mode="numpy"):
+    bins = np.linspace(min, max, steps)
+    bins = bins + (bins[1] - bins[0]) / 2
+    # if mode == "tf":
+    #     from tensorflow.ops import math_ops
+    #     func = math_ops._bucketize
+    #     bins = list(bins)
+    # else:
+    func = np.digitize
+
+    def digitilizer(x):
+        return func(x, bins)
+
+    return digitilizer
+
+
+def save_image_pil(path, data, *args, **kwargs):
+    from PIL import Image
+    if not isinstance(data, np.ndarray):
+        data = np.array(data)
+    if if_images(data):
+        data = draw_images2(data, *args, **kwargs)
+    if np.max(data) <= 1:
+        data = im_resolve(data)
+    if np.shape(data)[-1] == 1:
+        data = data[..., 0]
+    img = Image.fromarray(np.array(data, dtype="uint8"))
+    # test if black or wihte
+    if data.shape[-1] not in [3, 4]:
+        img = img.convert('L')
+    # todo maybe just check type for np.float
+    if path is None:
+        img.show()
+    else:
+        img.save(path)
+
+
+def save_image_pil2(path, data, *args, **kwargs):
+    from PIL import Image
+    if np.max(data) <= 1:
+        data = im_resolve(data)
+    if len(np.shape(data)) == 3 and np.shape(data)[-1] == 1:
+        data = data[..., 0]
+    img = Image.fromarray(data)
+    if len(data.shape) == 2 or (len(data.shape == 3) and data.shape[2] == 1):
+        img = img.convert('L')
+    # todo maybe just check type for np.float
+    if path is None:
+        img.show()
+    else:
+        img.save(path)
+
+
+#
+# https://towardsdatascience.com/how-to-create-a-gif-from-matplotlib-plots-in-python-6bec6c0c952c
+# duration, fps, loop
+def save_gif(path, data,sort_fn="first", *args, **kwargs):
+    data = get_paths_(data, sort_fn=sort_fn)
+    import imageio
+    with imageio.get_writer(path, mode='I', *args, **kwargs) as writer:
+        for filename in data:
+            image = imageio.imread(filename)
+            writer.append_data(image)
+
+
+# https://stackoverflow.com/questions/44947505/how-to-make-a-movie-out-of-images-in-python
+# https://docs.opencv.org/4.x/dd/d9e/classcv_1_1VideoWriter.html#ad59c61d8881ba2b2da22cff5487465b5
+# https://github.com/ContinuumIO/anaconda-issues/issues/223
+# https://softron.zendesk.com/hc/en-us/articles/207695697-List-of-FourCC-codes-for-video-codecs
+def save_video(path, data, *args, fps=1, fourcc="MJPG",sort_fn="first", **kwargs):
+    data = get_paths_(data, sort_fn=sort_fn)
+
+    import cv2
+
+    frame = cv2.imread(data[0])
+    height, width, layers = frame.shape
+
+    if isinstance(fourcc, str):
+        fourcc = cv2.VideoWriter_fourcc(*fourcc)
+    video = cv2.VideoWriter(path, fourcc, fps, (width, height))
+
+    for image in data:
+        video.write(cv2.imread(image))
+
+    cv2.destroyAllWindows()
+    video.release()
+
+
+def get_paths_(data, sort_fn="first"):
+    if isinstance(data, str):
+        data = list_dir(data, sort_fn=sort_fn)
+    return data
+
+
+def add_text(ax, text, pos=None, star_x=25, start_y=25, row_size=30):
+    text = lw(text)
+    if pos is None:
+        pos = [(star_x, start_y + i * row_size) for i in range(len(text))]
+    pos = lw(pos)
+    for i, t in enumerate(text):
+        # draw.text(pos[i], t,fill="black")
+        ax.text(pos[i][0], pos[i][1], t)
+
+
+@dir_decorator
+def save_image(path, data, description=None, size=None, dpi=None, *args, **kwargs):
+    if not isinstance(data, np.ndarray):
+        data = np.array(data)
+    fig = plt.figure()
+    if size is not None:
+        fig.set_size_inches(size)
+    # if title is not None:
+    #     plt.title(title)
+    ax = plt.Axes(fig, [0., 0., 1., 1.])
+    ax.set_axis_off()
+    fig.add_axes(ax)
+    if len(data.shape) == 2 or data.shape[-1] == 1:
+        # plt.set_cmap('Greys_r')
+        plt.set_cmap('Greys')
+    else:
+        plt.set_cmap('hot')
+    if if_images(data):
+        data = draw_images(data, *args, **kwargs)
+    ax.imshow(data, aspect='equal')
+    if description is not None:
+        add_text(ax, description, star_x=data.shape[1])
+
+    params = {
+        "fname": path,
+        "bbox_inches": 'tight',
+        "pad_inches": 0
+    }
+    if dpi is not None:
+        params['dpi'] = dpi
+    if path is None:
+        plt.show()
+    else:
+        plt.savefig(**params)
+    plt.clf()
+
+
+SAMPLE_IMAGES = {
+    "panda": "~/all/dataset/examples/panda.jpg",
+    "cat": "~/all/dataset/examples/cat.jpg",
+}
+
+
+def load_image(path=None, dtype="uint8", *args, **kwargs):
+    from tensorflow.keras.preprocessing import image
+    if path is None:
+        path = os.path.expanduser(SAMPLE_IMAGES["panda"])
+        if not os.path.isfile(path):
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            download_file("https://upload.wikimedia.org/wikipedia/commons/f/fe/Giant_Panda_in_Beijing_Zoo_1.JPG", path)
+        if not TARGET_SIZE in kwargs:
+            kwargs[TARGET_SIZE] = (224, 224)
+    elif path == "cat":
+        path = os.path.expanduser(SAMPLE_IMAGES["cat"])
+        kwargs[TARGET_SIZE] = (224, 224)
+
+    img = image.load_img(path, *args, **kwargs)
+    img = image.img_to_array(img, dtype=dtype)
+    return img
+
+
+def to_rgb(img, add_dim=True, check_dim=True):
+    shape = np.shape(img)
+    if add_dim and shape[-1] > 4:
+        img = img[..., None]
+    shape = np.shape(img)
+    if check_dim and shape[-1] != 1:
+        return img
+    return np.tile(img, reps=(1, 1, 3))
+
+# https://stackoverflow.com/questions/3844801/check-if-all-elements-in-a-list-are-identical
+def all_equal(iterable, key=None):
+    g = groupby(iterable, key=key)
+    return next(g, True) and not next(g, False)

raven_utils/params.py

@@ -1,110 +0,0 @@
-from dataclasses import dataclass
-from typing import Any, Tuple
-
-from ml_utils import get_str_name
-from grid_transformer.params import ImgSeqTransformerParameters
-from raven_utils import output
-
-from experiment_utils.parameters.nn_default import TP, EP
-
-
-@dataclass
-class SudokuParameters(ImgSeqTransformerParameters):
-    mask: str = "input"
-    col: int = 3
-    row: int = 3
-    pooling: int = 81
-    output_size: int = 9
-    size: int = 384
-
-
-@dataclass
-class RavenTransParameters(ImgSeqTransformerParameters):
-    mask: str = "last"
-    last_index: int = 8
-    col: int = 1
-    row: int = 1
-    output_size: int = output.SIZE
-    number_loss: bool = 0
-    pre: str = "images"
-    num_heads: int = 8
-
-
-MP = RavenTransParameters
-
-
-@dataclass
-class RavenSelectTransParameters(RavenTransParameters):
-    select_type: int = 2
-    select_out: int = 0
-    additional_out: int = 0
-    additional_copy: bool = True
-    tail_out: Tuple = (1000, 1000)
-    pre: str = "index"
-
-
-SMP = RavenSelectTransParameters
-
-from raven_utils.config.models import AVAILABLE_MODELS
-from experiment_utils.parameters.nn_clean import Parameters as BaseParameters
-from raven_utils.config.constant import RAVEN, LABELS, INDEX, FEATURES, RAV_METRICS, IMP_RAV_METRICS, ACC_NO_GROUP, \
-    ACC_SAME
-
-MODEL_NO = -1
-
-
-@dataclass
-class RavenParameters(BaseParameters):
-    dataset_name: str = RAVEN
-    data: Any = (
-        f"{dataset_name}/train.npy",
-        f"{dataset_name}/val.npy",
-        f"{dataset_name}/train_labels.npy",
-        f"{dataset_name}/val_labels.npy",
-        f"{dataset_name}/train_target.npy",
-        f"{dataset_name}/val_target.npy",
-        f"arr/train_features_{AVAILABLE_MODELS[MODEL_NO]}.npy",
-        f"arr/val_features_{AVAILABLE_MODELS[MODEL_NO]}.npy",
-        f"{dataset_name}/val_index.npy"
-        # DataParameters2()
-    )
-
-    # core_metrics: tuple = tuple(RAV_METRICS)
-    filter_metrics: tuple = tuple(IMP_RAV_METRICS)
-    # result_metric: str = ACC_NO_GROUP
-    result_metric: str = ACC_SAME
-
-    lw: float = 0.0001  # Autoencoder
-    loss_weight: float = 2.0
-    plw: int = 5.0
-    mp: RavenTransParameters = RavenTransParameters()
-
-    @property
-    def experiment(self):
-        # return "rav/trans"
-        return "rav/best_test3"
-        # return "rav/trans_weight"
-
-    # @property
-    # def name(self):
-    #     # return f"i{self.extractor}_{len(self.tail)}{self.tail[0]}_{self.type_}_{self.epsilon}_{self.last}_{self.epsilon_step}"
-    #     return f"{get_str_name(self.mp.pre)[0]}_{str(self.plw)[0]}_{str(self.mp.number_loss)[0]}_{self.mp.extractor}_{self.mp.noise if self.mp.noise else ''}_{self.mp.augmentation if self.mp.augmentation else ''}_{self.mp.extractor_shape}_{self.mp.no}_{self.mp.num_heads}_{self.mp.size}_{self.mp.pos_emd}_{self.mp.ff_mul}_{self.tp.batch}"
-
-
-@dataclass
-class BaselineRavenParameters(RavenParameters):
-
-    @property
-    def experiment(self):
-        # return "rav/best_test3"
-        return "rav/baseline"
-        # return "rav/trans_weight"
-
-    @property
-    def name(self):
-        # return f"i{self.extractor}_{len(self.tail)}{self.tail[0]}_{self.type_}_{self.epsilon}_{self.last}_{self.epsilon_step}"
-        return f"{get_str_name(self.mp.pre)[0]}_{str(self.plw)[0]}_{str(self.mp.number_loss)[0]}_{self.mp.extractor}_{self.mp.noise if self.mp.noise else ''}_{self.mp.augmentation if self.mp.augmentation else ''}_{self.mp.extractor_shape}_{self.mp.no}_{self.mp.num_heads}_{self.mp.size}_{self.mp.pos_emd}_{self.mp.ff_mul}_{self.tp.batch}"
-
-
-if __name__ == '__main__':
-    params = PreRavenTransParameters()

raven_utils/properties.py

@@ -1,6 +1,6 @@
-import raven_utils as rv
-from ml_utils import dict_from_list2, CalcDict
+
 import raven_utils.entity as entity
+from raven_utils.tools import dict_from_list2, CalcDict
 
 NAMES = [
     'Color',

raven_utils/render/rendering.py

@@ -6,10 +6,11 @@ import numpy as np
 from PIL import Image
 #
 # from AoT import Root
+from data_utils import Bag
+
 import raven_utils.decode
 from raven_utils.render.const import CENTER, DEFAULT_WIDTH, IMAGE_SIZE
 
-from data_utils import Bag
 
 from raven_utils.render_ import COLOR_VALUES, SIZE_VALUES, TYPE_VALUES, ANGLE_VALUES, RENDER_POSITIONS
 

raven_utils/rules.py

@@ -1,4 +1,5 @@
-from ml_utils import dict_from_list
+from raven_utils.tools import dict_from_list
+
 COMBINE = "Number/Position"
 
 ATTRIBUTES = [

raven_utils/tools.py

@@ -0,0 +1,274 @@
+import dataclasses
+import inspect
+import os
+import re
+import shutil
+from functools import partial
+from typing import Dict, Counter, Tuple
+import operator
+
+import numpy as np
+import requests
+from data_utils import ColumnIndexer, Dynamic
+from funcy import rcompose, identity
+from loguru import logger
+from tensorflow.keras import Model, Sequential
+from tensorflow.keras.layers import Layer
+from tensorflow.keras.losses import Loss
+
+
+def dir_decorator(fn):
+    def deco(path, *args, **kwargs):
+        if path is not None:
+            folder = os.path.dirname(path)
+            if folder:
+                os.makedirs(folder, exist_ok=True)
+        return fn(path, *args, **kwargs)
+
+    return deco
+
+def extract_args(args):
+    if il(args):
+        if il(args[0]):
+            return tuple(args[0])
+        return tuple(args)
+    return tuple(lw(args))
+
+
+def comp(*args):
+    return rcompose(*extract_args(args))
+
+
+def is_model(obj):
+    return isinstance(obj, (Model, Layer, Loss))
+
+def list_dir(root):
+    for path in os.listdir(root):
+        yield os.path.join(root, path)
+
+def is_int(obj, bool_=False):
+    if not bool_ and isinstance(obj, bool):
+        return False
+    return isinstance(obj, (int, np.integer))
+    # return issubclass(type(obj),int)
+
+
+def is_float(obj):
+    return isinstance(obj, (float, np.float))
+
+
+def is_num(obj, bool_=False):
+    return is_int(obj, bool_) or is_float(obj)
+
+
+def get_args(obj):
+    try:
+        return list(inspect.signature(obj).parameters.keys())
+    except Exception as e:
+        logger.error(f"{e}. Returning 1.")
+        return ["arg"]
+
+
+def il(data):
+    return isinstance(data, (list, tuple))
+
+
+def lw(data, none_empty=True, convert_tuple=True):
+    if isinstance(data, list):
+        return data
+    elif isinstance(data, tuple) and convert_tuple:
+        return list(data)
+    if none_empty and data is None:
+        return []
+    return [data]
+
+
+def dict_from_list(keys=None, values=None) -> Dict:
+    if values is None:
+        values = list(range(len(lw(keys))))
+    elif callable(values):
+        values = [values(k) for k in keys]
+    if keys is None:
+        keys = list(range(len(lw(values))))
+    elif callable(keys):
+        keys = [keys(v) for v in values]
+    if not isinstance(values, dict):
+        values = dict(zip(keys[:len(lw(values))], lw(values)))
+    return values
+
+
+def call(fn, x=None):
+    if len(get_args(fn)) > 0:
+        if isinstance(x, dict):
+            return fn(**x)
+        elif il(x):
+            return fn(*x)
+        else:
+            return fn(x)
+    else:
+        return fn()
+
+
+def get_str_comp(str_comp):
+    if callable(str_comp):
+        return str_comp
+    elif isinstance(str_comp, str):
+        if str_comp == "in":
+            return lambda x, y: x in y
+        elif str_comp in ["equal", "="]:
+            return lambda x, y: x == y
+        elif str_comp == "re":
+            return lambda x, y: re.match(x, y)
+    return lambda x, y: x == y
+
+
+def test(filters, x=None, str_comp="in"):
+    str_comp = get_str_comp(str_comp)
+    result = []
+    for f in lw(filters):
+        if isinstance(f, str):
+            result.append(str_comp(f, x))
+        elif callable(f):
+            result.append(call(f, x))
+        else:
+            result.append(f == x)
+    return result
+
+
+test_all = comp([test, all])
+test_any = comp([test, any])
+
+
+def filter_keys(d, keys, reverse=False, str_comp="=", *args, **kwargs):
+    fn = test_any
+    if dataclasses.is_dataclass(keys):
+        keys = dataclasses.asdict(keys)
+    if isinstance(keys, dict):
+        keys = list(keys.keys())
+    if dataclasses.is_dataclass(d):
+        d = dataclasses.asdict(d)
+    if reverse:
+        fn = comp(test_any, lambda x: not x)
+    return {k: v for k, v in d.items() if fn(keys, k, str_comp=str_comp, *args, **kwargs)}
+def none(x):
+    return [] if x is None else x
+
+def if_images(data):
+    if not hasattr(data, "shape"):
+        return False
+    shape = data.shape
+    return len(shape) > 3 or (len(shape) == 3 and shape[-1] > 4)
+
+
+def download_file(url, path=None):
+    if path is None:
+        path = url.split("/")[-1]
+
+    r = requests.get(url, stream=True)
+
+    if r.status_code == 200:
+        r.raw.decode_content = True
+
+        with open(path, 'wb') as f:
+            shutil.copyfileobj(r.raw, f)
+
+        logger.info(f'File {url} sucessfully downloaded to {path} ')
+    else:
+        logger.info(f'File {url} couldn\'t be retreived')
+
+@dataclasses.dataclass
+class ImageType:
+    mode: str = "RGB"
+    range_: Tuple = (0, 255)
+    channel: bool = True
+    reverse: bool = False
+    shape: bool = (None, 224, 224, 3)
+    dtype: str = "uint8"
+    counter: Counter = None
+
+    def __eq__(self, other):
+        return filter_keys(dataclasses.asdict(self), "counter", reverse=True) == filter_keys(dataclasses.asdict(other), "counter", reverse=True)
+
+
+def image_type(data, histogram=True):
+    data = np.asarray(data)
+    if not if_images(data):
+        return False
+    shape = data.shape
+    channel = True
+    range_ = data.min(), data.max()
+    counter = None
+    reverse = None
+    if shape[-1] == 3:
+        mode = "RGB"
+        if histogram:
+            counter = Counter(list(get_last_dim(data, 3).reshape(-1)))
+            reverse = counter.most_common(1)[0][0] == range_[1]
+    else:
+        mode = "L"
+        if shape[-1] != 1:
+            channel = False
+            if histogram:
+                counter = Counter(list(get_last_dim(data).reshape(-1)))
+                reverse = counter.most_common(1)[0][0] == range_[1]
+        else:
+            if histogram:
+                counter = Counter(list(get_last_dim(data, 3).reshape(-1)))
+                reverse = counter.most_common(1)[0][0] == range_[1]
+    return ImageType(mode, range_, channel, reverse, shape, data.dtype, counter)
+
+timage = partial(image_type, histogram=False)
+
+class OverrideDict(dict):
+    def to_dict(self):
+        # return {k: v for k, v in self.items()}
+        return dict(self)
+
+    def get_fn(self, index):
+        return self.__getitem__(index)
+
+
+class CalcDict(OverrideDict):
+    def operate(self, other, op, right=False):
+        if not hasattr(other, "__iter__"):
+            other = [other] * len(self)
+        if isinstance(other, dict):
+            other = other.values()
+        it = iter(other)
+        if right:
+            fn = lambda x, y: op(y, x)
+        else:
+            fn = op
+        return CalcDict({k: fn(v, next(it)) for k, v in self.items()})
+
+    def __add__(self, other):
+        return self.operate(other, operator.add)
+
+    def __sub__(self, other):
+        return self.operate(other, operator.sub)
+
+    def __mul__(self, other):
+        return self.operate(other, operator.mul)
+
+    def __truediv__(self, other):
+        return self.operate(other, operator.truediv)
+
+    def __radd__(self, other):
+        return self.operate(other, operator.add, right=True)
+
+    def __rsub__(self, other):
+        return self.operate(other, operator.sub, right=True)
+
+    def __rmul__(self, other):
+        return self.operate(other, operator.mul, right=True)
+
+    def __rtruediv__(self, other):
+        return self.operate(other, operator.truediv, right=True)
+
+
+def dict_from_list2(keys, values=None):
+    if values is None:
+        values = list(range(len(keys)))
+    return dict(zip(keys[:len(lw(values))], lw(values)))
+
+

raven_utils/uitls.py

@@ -1,24 +1,34 @@
 from functools import partial
-from itertools import product
 
 import numpy as np
-from funcy import identity
 
-from data_utils import gather, DataGenerator, Data
-from data_utils.sampling import DataSampler
-from models_utils import init_image as def_init_image, INPUTS, TARGET
+
+def gather(a, index):
+    return a[np.arange(np.shape(a)[0]), index]
+
 
 import raven_utils.group as group
 
-from data_utils import ops as D
+
+def def_init_image(shape=(10, 64, 64, 3), mode="uniform", min=0, max=1):
+    if mode == "normal" or mode == "n":
+        return np.random.normal(min, max, shape)
+    elif mode == "zero" or mode == 0:
+        return np.zeros(shape)
+    elif mode == "one" or mode == 1:
+        return np.ones(shape)
+    elif mode == "int" or isinstance(mode, int):
+        return np.random.randint(min, max, shape)
+    return np.random.uniform(min, max, shape)
+
 
 init_image = partial(def_init_image, shape=(16, 8, 80, 80, 1))
 
 
-def get_val_index(no=group.NO, base=3,add_end=False):
+def get_val_index(no=group.NO, base=3, add_end=False):
     indexes = np.arange(no) * 2000 + base
     if add_end:
-        indexes = np.concatenate([indexes, no*2000])
+        indexes = np.concatenate([indexes, no * 2000])
     return indexes
 
 
@@ -32,29 +42,6 @@ def get_matrix_from_data(x):
     return get_matrix(inputs, index)
 
 
-def get_data_class(data, batch_size=128):
-    fn = identity
-    shape = data[0].shape
-    train_generator = DataGenerator(
-        {
-            INPUTS: Data(data[0], fn),
-            TARGET: Data(data[2], fn),
-        },
-        sampler=DataSampler(np.array(list(product(np.arange(shape[0]), np.arange(shape[1]))))),
-        batch=batch_size
-    )
-    shape = data[1].shape
-    val_generator = DataGenerator(
-        {
-            INPUTS: Data(data[1], fn),
-            TARGET: Data(data[3], fn),
-        },
-        sampler=DataSampler(np.array(list(product(np.arange(shape[0]), np.arange(shape[1])))), shuffle=False),
-        batch=batch_size
-    )
-    return train_generator, val_generator
-
-
 def compare_from_result(result, data):
     data = data.data.data
     answer = D.gather(data['target'].data, data['index'].data[:, 0])

utils.py

@@ -1,14 +1,25 @@
 import numpy as np
-from data_utils.image import draw_images
-from ml_utils import il
+from funcy import identity
 
 import raven_utils as rv
+from raven_utils.constant import PROPERTY
+from raven_utils.decode import target_mask
+from raven_utils.image import draw_images
+from raven_utils.render.rendering import render_panels
+from raven_utils.tools import filter_keys, is_model, il
 from raven_utils.uitls import get_matrix
 from tensorflow.keras.models import load_model
 from raven_utils.draw import render_from_model
 import models
 import ast
 
+def render_from_model(data,predict,pre_fn=identity):
+    data = filter_keys(data, PROPERTY, reverse=True)
+    if is_model(predict) or str(type(predict)) == "<class 'tensorflow.python.saved_model.load.Loader._recreate_base_user_object.<locals>._UserObject'>":
+        predict = predict(data)
+    pro = np.array(target_mask(predict['predict_mask'].numpy()) * predict["predict"].numpy(), dtype=np.int8)
+    return pre_fn(render_panels(pro, target=False)[None])[0]
+
 
 def load_example(index=0):
     index = ast.literal_eval(str(index))