berlin demo

api.py

@@ -16,14 +16,16 @@ from types import SimpleNamespace
 from flask import Flask, request, send_from_directory
 from flask_cors import CORS
 from moviepy.editor import *
-from audiocraft.audiogen import AudioGen, audio_write
+from audiocraft.audiogen import AudioGen
 CACHE_DIR = 'flask_cache/'
-SOUNDSCAPE_DURATION = 6
+SOUNDSCAPE_DURATION = 4.4
 sound_generator = AudioGen.get_pretrained('facebook/audiogen-medium')
 sound_generator.set_generation_params(duration=SOUNDSCAPE_DURATION)
 print(f'{sound_generator.sample_rate=}')
 Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
 
+import nltk
+nltk.download('punkt')
 
 # SSH AGENT
 #   eval $(ssh-agent -s)
@@ -43,6 +45,7 @@ def _shift(x):
     return x  #* fade_in   # silence this
 
 def overlay(x, scene=None):
+    print(f'{scene=} OVERLAY')
     if scene is not None:
         
         # generate 4
@@ -51,7 +54,7 @@ def overlay(x, scene=None):
                                         [scene]
                                             )[0].detach().cpu().numpy()[0, :] for _ in range(1)]
         
-        print([j.shape for j in back], len(back), 'BACK')
+        #print([j.shape for j in back], len(back), 'BACK')
         
         # upsample to 24kHZ of StyleTTS
         print('Resampling')
@@ -78,7 +81,7 @@ def overlay(x, scene=None):
         back = _shift(back)
         print(f'\n====SOUND BACKGROUND SHAPE\n{back.shape=}',
               f'{np.abs(back.max())=}\n{x.shape=}')
-        x = .9 * x + .1 * back[:len(x)]
+        x = .1 * x + .9 * back[:len(x)]
     else:
         print('sound_background = None')
     return x
@@ -157,7 +160,7 @@ def serve_wav():
                 voice=r.get('voice')[0],
                 native=None if r.get('native') is None else CACHE_DIR + r.get('native')[0].replace("/",""),
                 affective = r.get('affective')[0],
-                scene=r.get('scene')[0]
+                scene=r.get('scene')[0] if r.get('scene') is not None else None 
                 )
     # print('\n==RECOMPOSED as \n',request.data,request.form,'\n==')
     

berlin_30_Oct_2024/lamma400b_children.txt

@@ -0,0 +1 @@
+Meet Georg Gisze, a super cool and rich merchant who lived a long time ago. Look at him sitting in his awesome office, wearing a bright red shirt and a fancy black cape - he looks like a superhero! Check out all the amazing things around him, like the colorful carpet, the shiny watch, and the beautiful glass vase. And have you noticed all the letters on the wall? Georg was a very busy man, and these letters show just how much work he had to do. Can you imagine what it would be like to be a merchant like Georg, traveling the world and collecting cool things? Maybe one day you'll have an amazing office like his too!

berlin_30_Oct_2024/lamma400b_professionals.txt

@@ -0,0 +1,10 @@
+Portrait of Georg Gisze, a paradigmatic exemplar of 16th-century mercantile
+portraiture, presents a sumptuously attired merchant amidst the accoutrements of
+his profession. The subject's opulent red silk sleeves and voluminous black cape,
+complemented by a beret, underscore his elevated social standing. The
+fastidiously rendered setting, replete with an exquisite Anatolian carpet and a
+delicate Venetian glass vase, serves as a visual testament to Gisze's considerable
+wealth and mercantile prowess. The incorporation of epistolary documents on the
+wall, meanwhile, provides a narrative anchor, emphasizing the subject's vocational identity as a merchant. Through the masterful deployment of symbolic objects and meticulous attention to detail, the artist crafts a nuanced portrait that
+not only captures the likeness of the sitter but also provides a fascinating glimpse
+into the rarefied world of Renaissance commerce.”

berlin_30_Oct_2024/lamma400b_visualimpair.txt

@@ -0,0 +1,5 @@
+A portrait of Georg Gisze, a wealthy merchant, sitting in his office.
+Georg Gisze is positioned centrally in the painting, seated in a three-quarter view.
+He wears a black cape draped over his right shoulder, with vibrant red silk sleeves visible underneath. His facial expression appears calm  focused. To his left, a delicate glass vase sits on a nearby shelf while a pocket watch lies
+open on the table in front of him. The table is covered with a patterned carpet that adds a pop of color to the scene.
+The walls behind Georg Gisze are adorned with numerous letters, some of which are folded, while others are open and appear to be in the process of being written. These letters serve as a testament to his profession as a merchant. The overall atmosphere of the painting is one of quiet contemplation, inviting the viewer to step into the serene and organized world of Georg Gisze's office.

berlin_30_Oct_2024/lamma70b_children.txt

@@ -0,0 +1,24 @@
+Meet Georg Gisze, a
+super cool and rich
+merchant from a long
+time ago. Look at him
+sitting in his fancy
+office, wearing a
+bright red shirt and a
+big, fancy black cape.
+Can you spot all the
+awesome things around him? There's a
+colorful carpet on the
+floor, a shiny watch on
+the table, and a
+beautiful glass vase.
+And check out all the
+letters on the wall -
+Georg must have been
+very busy with his
+work, sending and
+receiving messages
+from all around the
+world. What do you
+think Georg's job was
+like?”

berlin_30_Oct_2024/lamma70b_professionals.txt

@@ -0,0 +1,64 @@
+This exquisite oil on oak
+panel portrait,
+attributed to the
+renowned German
+artist Hans Holbein the
+Younger, presents a
+captivating
+representation of Georg
+Gisze, a prominent
+Hanseatic merchant. Executed with
+remarkable attention to
+detail, the painting
+showcases the
+subject's opulent attire,
+comprising crimson silk
+sleeves and a
+voluminous black cape,
+adorned with a beret,
+thereby underscoring
+his elevated social
+standing.
+The meticulously
+rendered setting,
+replete with symbols of
+Gisze's affluence and
+mercantile prowess,
+serves as a testament
+to the artist's mastery
+of still life composition.
+The incorporation of an
+Anatolian carpet, a
+Venetian glass vase,
+and a pocket watch not
+only demonstrates the
+subject's refined taste
+but also alludes to the
+burgeoning global trade
+networks of the
+Renaissance era.
+Furthermore, the
+presence of letters on
+the wall, presumably
+correspondence related
+to Gisze's mercantile
+activities, injects a
+sense of dynamism into
+the composition, while
+simultaneously highlighting the
+subject's profession
+and the concomitant
+responsibilities that
+accompany it. Through
+this portrait, Holbein
+offers a nuanced
+exploration of the
+intricate relationships
+between material
+wealth, social status,
+and personal identity,
+thereby providing a
+fascinating glimpse into
+the rarefied world of
+16th-century mercantile
+elites

berlin_30_Oct_2024/lamma70b_visualimpair.txt

@@ -0,0 +1,53 @@
+This 16th-century
+portrait painting
+depicts Georg Gisze,
+a prosperous
+merchant, seated in
+his office. He is
+dressed in attire that
+reflects his wealth,
+wearing vibrant red
+silk sleeves and a black cape draped
+over his shoulders.
+The subject is
+situated in a cluttered
+yet organized
+workspace,
+surrounded by
+various objects that
+highlight his
+profession and
+interests. A richly
+patterned carpet
+covers the floor, while
+a delicate glass vase
+sits atop a nearby
+surface, adding a
+touch of elegance to
+the space. A pocket
+watch is also visible,
+symbolizing the
+importance of time in
+Gisze's line of work.
+The walls of the
+office are adorned
+with letters, which
+serve as a testament
+to Gisze's occupation
+as a merchant. These
+letters likely represent
+correspondence with
+business associates,
+suppliers, or clients,
+providing insight into
+the merchant's daily
+activities.
+The overall
+atmosphere of the
+painting is one of refinement and
+sophistication,
+offering a glimpse
+into the life of a
+successful merchant
+during this time
+period.

berlin_30_Oct_2024/lamma8b_children.txt

@@ -0,0 +1,34 @@
+Hey kids, let's explore
+this cool painting
+together!
+Look at the rich man,
+Georg Gisze, sitting in
+his office. He's
+wearing a super cool
+red shirt and a big,
+fancy black cape. Can
+you spot the colorful
+carpet, the shiny
+watch, and the
+beautiful glass vase
+around him?
+Georg is a merchant,
+which means he buys
+and sells things.
+That's why you see
+lots of letters on the
+wall - he's busy
+working on his business!
+What do you think
+Georg is selling?
+Maybe it's something
+shiny, like the watch?
+Or maybe it's
+something colorful,
+like the carpet?
+Let's use our
+imagination and see
+what kind of
+adventures Georg
+might be having in his
+office!

berlin_30_Oct_2024/lamma8b_professionals.txt

@@ -0,0 +1,50 @@
+This captivating halflength portrait,
+attributed to the Dutch
+master Hans Holbein
+the Younger, presents a
+visually striking,
+representation of Georg
+Gisze, a prominent
+merchant, of the 16th
+century. The subject is
+depicted in a
+sumptuous red silk, doublet, adorned with
+voluminous black cape,
+and a characteristic
+beret, evoking an air, of
+opulence and
+refinement.
+The meticulously
+rendered background,
+replete with an
+intricately patterned Anatolian carpet, a
+delicate Venetian glass
+vase, and a precisioncrafted pocket watch,
+serves as a testament
+to Gisze's considerable
+wealth and status. The
+inclusion of letters on
+the wall, referencing his
+occupation as a
+merchant, underscores
+the subject's profession
+and reinforces the
+notion of his mercantile
+pursuits.
+Through this masterful
+portrayal, Holbein the
+Younger skillfully
+captures the essence
+of Gisze's person
+imbuing the subject
+with a sense of gravitas
+and authority. The
+painting's masterful
+execution,
+characterized by its
+precise attention, to
+detail and nuanced use
+of color, serves as a
+testament to the artists
+exceptional skill and
+artistry.

berlin_30_Oct_2024/lamma8b_visualimpair.txt

@@ -0,0 +1,65 @@
+Main Subject:
+The painting features
+a man, Georg Gisze,
+seated in his office.
+He is dressed in a
+distinctive outfit,
+which includes red
+silk sleeves and a
+black cape. This attire
+suggests his wealth
+and status as a
+merchant.
+Background:
+The room is filled with
+various objects that
+reflect Georg's
+profession. You'll
+notice a patterned
+carpet beneath his
+feet, which adds a
+sense of luxury to the
+space. On a nearby table, a delicate glass
+vase sits elegantly,
+showcasing the
+craftsmanship of the
+era. A pocket watch
+hangs from his waist,
+symbolizing the
+importance of
+timekeeping in
+business.
+Visual Elements:
+The walls of the
+office are adorned
+with letters, which are
+a testament to
+Georg's work as a
+merchant. These
+letters likely contain
+important business
+correspondence,
+highlighting his role in
+the trade.
+Additional Details:
+As you imagine this
+scene, consider the
+atmosphere of a 16thcentury merchant's
+office. The painting
+invites you to step
+into Georg's world,
+where commerce,
+and the values of his
+time.
+Accessibility
+Features:
+This description is
+designed to be
+accessible to visually impaired individuals.
+The text is structured
+to provide a clear
+understanding of the
+image, with a focus
+on the main subject,
+background, and
+visual elements.

berlin_demo.py

@@ -0,0 +1,31 @@
+# make berlin wavs
+import subprocess
+
+BERLIN_DIR = 'berlin_30_Oct_2024/'
+
+pairs = [
+  ['en_US/hifi-tts_low#9017', 'lamma8b_children.txt', False],
+  ['en_US/hifi-tts_low#9017' , 'lamma70b_children.txt', False],
+  ['en_US/hifi-tts_low#9017' , 'lamma400b_children.txt', False],
+  ['en_US/cmu-arctic_low#jmk', 'lamma8b_professionals.txt', True],
+  ['en_US/cmu-arctic_low#jmk', 'lamma70b_professionals.txt', True],
+  ['en_US/cmu-arctic_low#jmk', 'lamma400b_professionals.txt', True],
+  ['en_US/cmu-arctic_low#eey'   , 'lamma70b_visualimpair.txt', True],
+  ['en_US/cmu-arctic_low#eey'   , 'lamma8b_visualimpair.txt', True],
+  ['en_US/cmu-arctic_low#eey'   , 'lamma400b_visualimpair.txt', True],
+]
+
+for vox, text_file, affect in pairs:
+	print(vox, text_file)
+	subprocess.run(
+	        [
+	         "python",
+	         "tts.py",
+	         "--text", BERLIN_DIR + text_file,
+	         # "--affect",
+	         #'--image', '_tmp_banner.png',
+	         # '--scene', 'calm sounds of castle',
+	         '--voice', vox,
+	         '--out_file', text_file[:-4],  # save to correct location is handled in client
+	            ])
+

demo.py

@@ -7,7 +7,7 @@ print('\n\n\n\n___________________')
 txt = 'dogs in street'
 
 sound_generator = AudioGen.get_pretrained('facebook/audiogen-medium')
-sound_generator.set_generation_params(duration=1.7)   # why is generating so long at 14 seconds
+sound_generator.set_generation_params(duration=.7)   # why is generating so long at 14 seconds
 
 x = sound_generator.generate([txt])[0].detach().cpu().numpy()[0, :]
 x /= np.abs(x).max() + 1e-7

tts.py

@@ -3,6 +3,8 @@ import numpy as np
 import argparse
 import os
 import requests
+from pathlib import Path
+Path('out/').mkdir(parents=True, exist_ok=True)
 
 # SSH AGENT
 #   eval $(ssh-agent -s)
@@ -71,7 +73,7 @@ def command_line_args():
         '--scene',
         help='Sound scene description.',
         type=str,
-        default='calm background sounds of a castle'
+        default=None, #'calm background sounds of a castle'
     )
     return parser
 
@@ -148,7 +150,7 @@ def cli():
     
     with open(
         # args.out_file is not send to server - server writes tmp - copied by client
-        args.out_file + '.' + response.headers['suffix-file-type'].split('.')[-1],
+        './out/' + args.out_file + '.' + response.headers['suffix-file-type'].split('.')[-1],
         'wb'
         ) as f:
         f.write(response.content)