{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Copyright 2020 Erik Härkönen. All rights reserved.\n",
"# This file is licensed to you under the Apache License, Version 2.0 (the \"License\");\n",
"# you may not use this file except in compliance with the License. You may obtain a copy\n",
"# of the License at http://www.apache.org/licenses/LICENSE-2.0\n",
"\n",
"# Unless required by applicable law or agreed to in writing, software distributed under\n",
"# the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS\n",
"# OF ANY KIND, either express or implied. See the License for the specific language\n",
"# governing permissions and limitations under the License.\n",
"\n",
"# Teaser: sequence of 3 interesting edits\n",
"%matplotlib inline\n",
"from notebook_init import *\n",
"\n",
"rand = lambda : np.random.randint(np.iinfo(np.int32).max)\n",
"outdir = Path('out/figures/teaser')\n",
"makedirs(outdir, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def setup_model(model_name, class_name, layer_name):\n",
" global inst, model, lat_comp, lat_mean, lat_std\n",
"\n",
" use_w = 'StyleGAN' in model_name\n",
" inst = get_instrumented_model(model_name, class_name, layer_name, device, use_w=use_w, inst=inst)\n",
" model = inst.model\n",
"\n",
" pc_config = Config(components=80, n=1_000_000, batch_size=200,\n",
" layer=layer_name, model=model_name, output_class=class_name, use_w=use_w)\n",
" dump_name = get_or_compute(pc_config, inst)\n",
"\n",
" with np.load(dump_name) as data:\n",
" lat_comp = data['lat_comp']\n",
" lat_mean = data['lat_mean']\n",
" lat_std = data['lat_stdev']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def perform_edit(seeds, edit_sequence, save_images=False, crop=None):\n",
" max_figs = 1000 if save_images else 10\n",
"\n",
" for seed in seeds[:max_figs]:\n",
" w = model.sample_latent(1, seed=seed).cpu().numpy()\n",
" w = [w]*model.get_max_latents()\n",
" imgs = []\n",
" \n",
" # Starting point\n",
" imgs.append(model.sample_np(w))\n",
" \n",
" # Perform edits in order\n",
" for edit in edit_sequence:\n",
" (idx, start, end, strength, invert) = configs[edit]\n",
" \n",
" # Find out coordinate of w along PC\n",
" w_centered = w[0] - lat_mean\n",
" w_coord = np.sum(w_centered.reshape(-1)*lat_comp[idx].reshape(-1)) / lat_std[idx]\n",
" \n",
" # Invert property if desired (e.g. flip rotation)\n",
" # Otherwise reinforce existing\n",
" if invert:\n",
" sign = w_coord / np.abs(w_coord)\n",
" target = -sign*strength # opposite side of mean\n",
" else:\n",
" target = strength\n",
" \n",
" delta = target - w_coord # offset vector\n",
" \n",
" for l in range(start, end):\n",
" w[l] = w[l] + lat_comp[idx]*lat_std[idx]*delta\n",
" imgs.append(model.sample_np(w))\n",
" \n",
" # Crop away black borders\n",
" if crop:\n",
" imgs = [img[crop[0]:-crop[1], crop[2]:-crop[3], :] for img in imgs]\n",
"\n",
" if save_images:\n",
" # Save to disk\n",
" for i, img in enumerate(imgs):\n",
" Image.fromarray((img*255).astype(np.uint8)).save(outdir / f'teaser_{seed}_{i}.png')\n",
" \n",
" # Show in notebook\n",
" strip = np.hstack(imgs)\n",
" #strip = strip[::2, ::2, :] # 2x downscale for preview\n",
" plt.figure(figsize=(30,5))\n",
" plt.imshow(strip, interpolation='bilinear')\n",
" plt.axis('off')\n",
" plt.show()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# (idx, edit_start, edit_end, strength, invert)\n",
"configs = {\n",
" # StyleGAN2 cars W\n",
" 'Redness': (22, 9, 11, -8, False),\n",
" 'Horizontal flip': ( 0, 0, 5, 2.0, True),\n",
" 'Add grass': (41, 9, 11, -18, False),\n",
" 'Blocky shape': (16, 3, 6, 25, False),\n",
"\n",
" # BigGAN-512 irish_setter\n",
" 'Move right': ( 0, 0, 15, -1.5, False),\n",
" 'Rotate': ( 3, 0, 15, -0.5, False),\n",
" 'Move back': ( 4, 0, 15, 2.5, False),\n",
" 'Zoom in': ( 6, 0, 15, -2.0, False),\n",
" 'Zoom out': (12, 0, 15, -4.0, False),\n",
" 'Sharpen BG': (13, 6, 9, 20.0, False),\n",
" 'Camera down': (15, 1, 6, -4.0, False),\n",
" 'Light right': (28, 7, 8, 30, False),\n",
" 'Pixelate': (46, 10, 11, -25, False),\n",
" 'Reeds': (61, 4, 8, -15, False),\n",
" 'Dry BG': (65, 6, 8, -30, False),\n",
" 'Grass length': (69, 5, 8, 15, False),\n",
"\n",
" # StyleGAN2 ffhq\n",
" 'frizzy_hair': (31, 2, 6, 20, False),\n",
" 'background_blur': (49, 6, 9, 20, False),\n",
" 'bald': (21, 2, 5, 20, False),\n",
" 'big_smile': (19, 4, 5, 20, False),\n",
" 'caricature_smile': (26, 3, 8, 13, False),\n",
" 'scary_eyes': (33, 6, 8, 20, False),\n",
" 'curly_hair': (47, 3, 6, 20, False),\n",
" 'dark_bg_shiny_hair': (13, 8, 9, 20, False),\n",
" 'dark_hair_and_light_pos': (14, 8, 9, 20, False),\n",
" 'dark_hair': (16, 8, 9, 20, False),\n",
" 'disgusted': (43, 6, 8, -30, False),\n",
" 'displeased': (36, 4, 7, 20, False),\n",
" 'eye_openness': (54, 7, 8, 20, False),\n",
" 'eye_wrinkles': (28, 6, 8, 20, False),\n",
" 'eyebrow_thickness': (37, 8, 9, 20, False),\n",
" 'face_roundness': (37, 0, 5, 20, False),\n",
" 'fearful_eyes': (54, 4, 10, 20, False),\n",
" 'hairline': (21, 4, 5, -20, False),\n",
" 'happy_frizzy_hair': (30, 0, 8, 20, False),\n",
" 'happy_elderly_lady': (27, 4, 7, 20, False),\n",
" 'head_angle_up': (11, 1, 4, 20, False),\n",
" 'huge_grin': (28, 4, 6, 20, False),\n",
" 'in_awe': (23, 3, 6, -15, False),\n",
" 'wide_smile': (23, 3, 6, 20, False),\n",
" 'large_jaw': (22, 3, 6, 20, False),\n",
" 'light_lr': (15, 8, 9, 10, False),\n",
" 'lipstick_and_age': (34, 6, 11, 20, False),\n",
" 'lipstick': (34, 10, 11, 20, False),\n",
" 'mascara_vs_beard': (41, 6, 9, 20, False),\n",
" 'nose_length': (51, 4, 5, -20, False),\n",
" 'elderly_woman': (34, 6, 7, 20, False),\n",
" 'overexposed': (27, 8, 18, 15, False),\n",
" 'screaming': (35, 3, 7, -15, False),\n",
" 'short_face': (32, 2, 6, -20, False),\n",
" 'show_front_teeth': (59, 4, 5, 40, False),\n",
" 'smile': (46, 4, 5, -20, False),\n",
" 'straight_bowl_cut': (20, 4, 5, -20, False),\n",
" 'sunlight_in_face': (10, 8, 9, 10, False),\n",
" 'trimmed_beard': (58, 7, 9, 20, False),\n",
" 'white_hair': (57, 7, 10, -24, False),\n",
" 'wrinkles': (20, 6, 7, -18, False),\n",
" 'boyishness': (8, 2, 5, 20, False),\n",
"}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# StyleGAN2 faces - emphasis on novel edits\n",
"setup_model('StyleGAN2', 'ffhq', 'style')\n",
"model.truncation = 0.7\n",
"model.use_w()\n",
"\n",
"seeds = [6293435, 2105448342] # + [rand() for _ in range(1)]\n",
"print(seeds)\n",
"edits = ['wrinkles', 'white_hair', 'in_awe', 'overexposed']\n",
"perform_edit(seeds, edits, True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# StyleGAN2 cars\n",
"setup_model('StyleGAN2', 'car', 'style')\n",
"model.truncation = 0.6\n",
"model.use_w()\n",
"\n",
"seeds = [440749230] # + [rand() for _ in range(10)]\n",
"edits = ['Redness', 'Horizontal flip', 'Add grass', 'Blocky shape']\n",
"perform_edit(seeds, edits, True, crop=[64, 64, 1, 1])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# BigGAN-512 irish setter\n",
"setup_model('BigGAN-512', 'husky', 'generator.gen_z')\n",
"model.set_output_class('irish_setter')\n",
"model.truncation = 0.6\n",
"\n",
"seeds = [489408325]# + [rand() for _ in range(10)]\n",
"edits = ['Rotate', 'Zoom out', 'Camera down', 'Reeds']\n",
"perform_edit(seeds, edits, True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}