update component's frontend

Now the component is able to display each sources and corresponding
annotation separately

sourceviewer/frontend/Index.svelte

@@ -10,13 +10,13 @@
 	import InteractiveAudio from "./interactive/InteractiveAudio.svelte";
 	import { StatusTracker } from "@gradio/statustracker";
 	import { Block, UploadText } from "@gradio/atoms";
-	import type { WaveformOptions } from "./shared/types";
+	import type { WaveformOptions, Segment } from "./shared/types";
 
 	export let elem_id = "";
 	export let elem_classes: string[] = [];
 	export let visible = true;
 	export let interactive: boolean;
-	export let value: null | FileData = null;
+	export let value: null | {"segments": Segment[], "sources_file": FileData} = null;
 	export let sources:
 		| ["microphone"]
 		| ["upload"]
@@ -54,11 +54,11 @@
 		share: ShareData;
 	}>;
 
-	let old_value: null | FileData = null;
+	let old_value: null | {"segments": Segment[], "sources_file": FileData} = null;
 
 	let active_source: "microphone" | "upload";
 
-	let initial_value: null | FileData = value;
+	let initial_value: null | {"segments": Segment[], "sources_file": FileData} = value;
 
 	$: if (value && initial_value === null) {
 		initial_value = value;
@@ -105,23 +105,10 @@
 		normalize: true,
 		minPxPerSec: 20,
 		mediaControls: waveform_options.show_controls,
-		sampleRate: waveform_options.sample_rate || 44100
+		sampleRate: waveform_options.sample_rate || 44100,
+		splitChannels: true,
 	};
 
-	const trim_region_settings = {
-		color: waveform_options.trim_region_color,
-		drag: true,
-		resize: true
-	};
-
-	function set_trim_region_colour(): void {
-		document.documentElement.style.setProperty(
-			"--trim-region-color",
-			trim_region_settings.color || color_accent
-		);
-	}
-
-	set_trim_region_colour();
 
 	function handle_error({ detail }: CustomEvent<string>): void {
 		const [level, status] = detail.includes("Invalid file type")
@@ -160,6 +147,7 @@
 			{show_share_button}
 			{value}
 			{label}
+			{root}
 			{waveform_settings}
 			{waveform_options}
 			{editable}
@@ -220,7 +208,6 @@
 			i18n={gradio.i18n}
 			{waveform_settings}
 			{waveform_options}
-			{trim_region_settings}
 		>
 			<UploadText i18n={gradio.i18n} type="audio" />
 		</InteractiveAudio>

sourceviewer/frontend/interactive/InteractiveAudio.svelte

@@ -18,7 +18,7 @@
 	import { SelectSource } from "@gradio/atoms";
 	import type { WaveformOptions } from "../shared/types";
 
-	export let value: null | FileData = null;
+	export let value: null | {"segments": Segment[], "sources_file": FileData} = null;
 	export let label: string;
 	export let root: string;
 	export let show_label = true;
@@ -32,7 +32,6 @@
 	export let streaming = false;
 	export let i18n: I18nFormatter;
 	export let waveform_settings: Record<string, any>;
-	export let trim_region_settings = {};
 	export let waveform_options: WaveformOptions = {};
 	export let dragging: boolean;
 	export let active_source: "microphone" | "upload";
@@ -74,7 +73,7 @@
 	}
 
 	const dispatch = createEventDispatcher<{
-		change: FileData | null;
+		change: typeof value;
 		stream: FileData;
 		edit: never;
 		play: never;
@@ -96,7 +95,7 @@
 	): Promise<void> => {
 		let _audio_blob = new File(blobs, "audio.wav");
 		const val = await prepare_files([_audio_blob], event === "stream");
-		value = (
+		value.sources_file = (
 			(await upload(val, root, undefined, upload_fn))?.filter(
 				Boolean
 			) as FileData[]
@@ -193,8 +192,9 @@
 	}
 
 	function handle_load({ detail }: { detail: FileData }): void {
-		value = detail;
-		dispatch("change", detail);
+		value = {"segments": [], "sources_file": null}
+		value.sources_file = detail;
+		dispatch("change", value);
 		dispatch("upload", detail);
 	}
 
@@ -264,7 +264,7 @@
 			{i18n}
 			on:clear={clear}
 			on:edit={() => (mode = "edit")}
-			download={show_download_button ? value.url : null}
+			download={show_download_button ? value.sources_file.url : null}
 			absolute={true}
 		/>
 
@@ -272,11 +272,11 @@
 			bind:mode
 			{value}
 			{label}
+			{root}
 			{i18n}
 			{dispatch_blob}
 			{waveform_settings}
 			{waveform_options}
-			{trim_region_settings}
 			{handle_reset_value}
 			{editable}
 			interactive

sourceviewer/frontend/package-lock.json

@@ -22,7 +22,7 @@
         "extendable-media-recorder-wav-encoder": "^7.0.76",
         "resize-observer-polyfill": "^1.5.1",
         "svelte-range-slider-pips": "^2.0.1",
-        "wavesurfer.js": "^7.4.2"
+        "wavesurfer.js": "^7.8.9"
       }
     },
     "node_modules/@ampproject/remapping": {
@@ -1258,9 +1258,10 @@
       "integrity": "sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ=="
     },
     "node_modules/wavesurfer.js": {
-      "version": "7.8.8",
-      "resolved": "https://registry.npmjs.org/wavesurfer.js/-/wavesurfer.js-7.8.8.tgz",
-      "integrity": "sha512-FNiJ6RCAu4TijRcwdaqlN19+Y+Hhqb7Nfo2P4LudzhNnQRMKvF3QmXd9+KHlQZrSjexeK3WbY4COsxgGJkxMag=="
+      "version": "7.8.9",
+      "resolved": "https://registry.npmjs.org/wavesurfer.js/-/wavesurfer.js-7.8.9.tgz",
+      "integrity": "sha512-d7jDSrzNp3UErXWEKFlIYRdvzaMlxCoM6Or4qR8uURFp9smcxLXS81uatcs6aPX1OSJgEyyBuffMR5Io3Ft2FQ==",
+      "license": "BSD-3-Clause"
     },
     "node_modules/worker-factory": {
       "version": "7.0.34",

sourceviewer/frontend/package.json

@@ -15,12 +15,12 @@
     "@gradio/upload": "0.8.5",
     "@gradio/utils": "0.3.2",
     "@gradio/wasm": "0.10.0",
+    "@types/wavesurfer.js": "^6.0.10",
     "extendable-media-recorder": "^9.0.0",
     "extendable-media-recorder-wav-encoder": "^7.0.76",
     "resize-observer-polyfill": "^1.5.1",
     "svelte-range-slider-pips": "^2.0.1",
-    "wavesurfer.js": "^7.4.2",
-    "@types/wavesurfer.js": "^6.0.10"
+    "wavesurfer.js": "^7.8.9"
   },
   "main_changeset": true,
   "main": "index.ts",
@@ -30,4 +30,4 @@
     "./shared": "./shared/index.ts",
     "./package.json": "./package.json"
   }
-}
+}

sourceviewer/frontend/player/AudioPlayer.svelte

@@ -3,17 +3,19 @@
 	import { Music } from "@gradio/icons";
 	import { format_time, type I18nFormatter } from "@gradio/utils";
 	import WaveSurfer from "wavesurfer.js";
+	import RegionsPlugin, {type Region} from "wavesurfer.js/dist/plugins/regions";
 	import { skip_audio, process_audio } from "../shared/utils";
 	import WaveformControls from "../shared/WaveformControls.svelte";
 	import { Empty } from "@gradio/atoms";
 	import { resolve_wasm_src } from "@gradio/wasm/svelte";
 	import type { FileData } from "@gradio/client";
-	import type { WaveformOptions } from "../shared/types";
+	import type { WaveformOptions, Segment } from "../shared/types";
 	import { createEventDispatcher } from "svelte";
 
-	export let value: null | FileData = null;
-	$: url = value?.url;
+	export let value: null | {"segments": Segment[], "sources_file": FileData}= null;
+	$: url = value?.sources_file.url;
 	export let label: string;
+	export let root: string;
 	export let i18n: I18nFormatter;
 	export let dispatch_blob: (
 		blobs: Uint8Array[] | Blob[],
@@ -21,7 +23,6 @@
 	) => Promise<void> = () => Promise.resolve();
 	export let interactive = false;
 	export let editable = true;
-	export let trim_region_settings = {};
 	export let waveform_settings: Record<string, any>;
 	export let waveform_options: WaveformOptions;
 	export let mode = "";
@@ -29,12 +30,15 @@
 
 	let container: HTMLDivElement;
 	let waveform: WaveSurfer | undefined;
+	let wsRegion: RegionsPlugin | undefined;
 	let playing = false;
 
 	let timeRef: HTMLTimeElement;
 	let durationRef: HTMLTimeElement;
 	let audio_duration: number;
 
+	let colors: string[] = ["red", "green", "blue", "yellow", "magenta", "cyan"];
+
 	let trimDuration = 0;
 
 	let show_volume_slider = false;
@@ -52,11 +56,7 @@
 			container: container,
 			...waveform_settings
 		});
-		resolve_wasm_src(value?.url).then((resolved_src) => {
-			if (resolved_src && waveform) {
-				return waveform.load(resolved_src);
-			}
-		});
+		waveform.load(root + `/file=${value.sources_file.path}`)
 	};
 
 	$: if (container !== undefined) {
@@ -69,6 +69,23 @@
 	$: waveform?.on("decode", (duration: any) => {
 		audio_duration = duration;
 		durationRef && (durationRef.textContent = format_time(duration));
+
+		if(!wsRegion){
+			wsRegion = waveform.registerPlugin(RegionsPlugin.create())
+			value.segments.forEach(segment => {
+				const region = wsRegion.addRegion({
+					start: segment.start,
+					end: segment.end,
+					channelIdx: segment.channel,
+					drag: false,
+					resize: false,
+					color: colors[segment.channel % colors.length],
+				});
+				console.log(region.color)
+				const regionHeight = 100 / waveform.getDecodedData().numberOfChannels;
+				region.element.style.cssText += `height: ${regionHeight}% !important;`
+			});
+		}
 	});
 
 	$: waveform?.on(
@@ -120,7 +137,7 @@
 
 	async function load_audio(data: string): Promise<void> {
 		await resolve_wasm_src(data).then((resolved_src) => {
-			if (!resolved_src || value?.is_stream) return;
+			if (!resolved_src || value?.sources_file.is_stream) return;
 			return waveform?.load(resolved_src);
 		});
 	}
@@ -143,10 +160,10 @@
 	<Empty size="small">
 		<Music />
 	</Empty>
-{:else if value.is_stream}
+{:else if value.sources_file.is_stream}
 	<audio
 		class="standard-player"
-		src={value.url}
+		src={value.sources_file.url}
 		controls
 		autoplay={waveform_settings.autoplay}
 	/>
@@ -171,20 +188,17 @@
 
 		{#if waveform}
 			<WaveformControls
-				{container}
 				{waveform}
 				{playing}
 				{audio_duration}
 				{i18n}
 				{interactive}
-				{handle_trim_audio}
 				bind:mode
 				bind:trimDuration
 				bind:show_volume_slider
 				show_redo={interactive}
 				{handle_reset_value}
 				{waveform_options}
-				{trim_region_settings}
 				{editable}
 			/>
 		{/if}

sourceviewer/frontend/shared/WaveformControls.svelte

@@ -16,12 +16,9 @@
 	export let playing: boolean;
 	export let show_redo = false;
 	export let interactive = false;
-	export let handle_trim_audio: (start: number, end: number) => void;
 	export let mode = "";
-	export let container: HTMLDivElement;
 	export let handle_reset_value: () => void;
 	export let waveform_options: WaveformOptions = {};
-	export let trim_region_settings: WaveformOptions = {};
 	export let show_volume_slider = false;
 	export let editable = true;
 
@@ -30,94 +27,10 @@
 	let playbackSpeeds = [0.5, 1, 1.5, 2];
 	let playbackSpeed = playbackSpeeds[1];
 
-	let trimRegion: RegionsPlugin;
 	let activeRegion: Region | null = null;
 
-	let leftRegionHandle: HTMLDivElement | null;
-	let rightRegionHandle: HTMLDivElement | null;
-	let activeHandle = "";
-
 	let currentVolume = 1;
 
-	$: trimRegion = waveform.registerPlugin(RegionsPlugin.create());
-
-	$: trimRegion?.on("region-out", (region) => {
-		region.play();
-	});
-
-	$: trimRegion?.on("region-updated", (region) => {
-		trimDuration = region.end - region.start;
-	});
-
-	$: trimRegion?.on("region-clicked", (region, e) => {
-		e.stopPropagation(); // prevent triggering a click on the waveform
-		activeRegion = region;
-		region.play();
-	});
-
-	const addTrimRegion = (): void => {
-		activeRegion = trimRegion.addRegion({
-			start: audio_duration / 4,
-			end: audio_duration / 2,
-			...trim_region_settings
-		});
-
-		trimDuration = activeRegion.end - activeRegion.start;
-	};
-
-	$: if (activeRegion) {
-		const shadowRoot = container.children[0]!.shadowRoot!;
-
-		rightRegionHandle = shadowRoot.querySelector('[data-resize="right"]');
-		leftRegionHandle = shadowRoot.querySelector('[data-resize="left"]');
-
-		if (leftRegionHandle && rightRegionHandle) {
-			leftRegionHandle.setAttribute("role", "button");
-			rightRegionHandle.setAttribute("role", "button");
-			leftRegionHandle?.setAttribute("aria-label", "Drag to adjust start time");
-			rightRegionHandle?.setAttribute("aria-label", "Drag to adjust end time");
-			leftRegionHandle?.setAttribute("tabindex", "0");
-			rightRegionHandle?.setAttribute("tabindex", "0");
-
-			leftRegionHandle.addEventListener("focus", () => {
-				if (trimRegion) activeHandle = "left";
-			});
-
-			rightRegionHandle.addEventListener("focus", () => {
-				if (trimRegion) activeHandle = "right";
-			});
-		}
-	}
-
-	const trimAudio = (): void => {
-		if (waveform && trimRegion) {
-			if (activeRegion) {
-				const start = activeRegion.start;
-				const end = activeRegion.end;
-				handle_trim_audio(start, end);
-				mode = "";
-				activeRegion = null;
-			}
-		}
-	};
-
-	const clearRegions = (): void => {
-		trimRegion?.getRegions().forEach((region) => {
-			region.remove();
-		});
-		trimRegion?.clearRegions();
-	};
-
-	const toggleTrimmingMode = (): void => {
-		clearRegions();
-		if (mode === "edit") {
-			mode = "";
-		} else {
-			mode = "edit";
-			addTrimRegion();
-		}
-	};
-
 	const adjustRegionHandles = (handle: string, key: string): void => {
 		let newStart;
 		let newEnd;
@@ -149,14 +62,6 @@
 		trimDuration = activeRegion.end - activeRegion.start;
 	};
 
-	$: trimRegion &&
-		window.addEventListener("keydown", (e) => {
-			if (e.key === "ArrowLeft") {
-				adjustRegionHandles(activeHandle, "ArrowLeft");
-			} else if (e.key === "ArrowRight") {
-				adjustRegionHandles(activeHandle, "ArrowRight");
-			}
-		});
 </script>
 
 <div class="controls" data-testid="waveform-controls">
@@ -246,27 +151,12 @@
 					aria-label="Reset audio"
 					on:click={() => {
 						handle_reset_value();
-						clearRegions();
 						mode = "";
 					}}
 				>
 					<Undo />
 				</button>
 			{/if}
-
-			{#if mode === ""}
-				<button
-					class="action icon"
-					aria-label="Trim audio to selection"
-					on:click={toggleTrimmingMode}
-				>
-					<Trim />
-				</button>
-			{:else}
-				<button class="text-button" on:click={trimAudio}>Trim</button>
-				<button class="text-button" on:click={toggleTrimmingMode}>Cancel</button
-				>
-			{/if}
 		{/if}
 	</div>
 </div>
@@ -278,24 +168,6 @@
 		align-items: center;
 		grid-area: editing;
 	}
-	.text-button {
-		border: 1px solid var(--neutral-400);
-		border-radius: var(--radius-sm);
-		font-weight: 300;
-		font-size: var(--size-3);
-		text-align: center;
-		color: var(--neutral-400);
-		height: var(--size-5);
-		font-weight: bold;
-		padding: 0 5px;
-		margin-left: 5px;
-	}
-
-	.text-button:hover,
-	.text-button:focus {
-		color: var(--color-accent);
-		border-color: var(--color-accent);
-	}
 
 	.controls {
 		display: grid;

sourceviewer/frontend/shared/WaveformRecordControls.svelte

@@ -245,7 +245,6 @@
 
 	:global(::part(region)) {
 		border-radius: var(--radius-md);
-		height: 98% !important;
 		border: 1px solid var(--trim-region-color);
 		background-color: unset;
 		border-width: 1px 3px;
@@ -258,7 +257,6 @@
 		left: 0;
 		width: 100%;
 		height: 100%;
-		background: var(--trim-region-color);
 		opacity: 0.2;
 		border-radius: var(--radius-md);
 	}

sourceviewer/frontend/shared/types.ts

@@ -7,3 +7,9 @@ export type WaveformOptions = {
 	show_recording_waveform?: boolean;
 	sample_rate?: number;
 };
+
+export type Segment = {
+	start: number;
+	end: number;
+	channel: number;
+}

sourceviewer/frontend/static/StaticAudio.svelte

@@ -8,10 +8,11 @@
 	import { createEventDispatcher } from "svelte";
 	import type { FileData } from "@gradio/client";
 	import { DownloadLink } from "@gradio/wasm/svelte";
-	import type { WaveformOptions } from "../shared/types";
+	import type { WaveformOptions, Segment } from "../shared/types";
 
-	export let value: null | FileData = null;
+	export let value: null | {"segments": Segment[], "sources_file": FileData} = null;
 	export let label: string;
+	export let root: string;
 	export let show_label = true;
 	export let show_download_button = true;
 	export let show_share_button = false;
@@ -21,7 +22,7 @@
 	export let editable = true;
 
 	const dispatch = createEventDispatcher<{
-		change: FileData;
+		change: typeof value;
 		play: undefined;
 		pause: undefined;
 		end: undefined;
@@ -41,7 +42,7 @@
 {#if value !== null}
 	<div class="icon-buttons">
 		{#if show_download_button}
-			<DownloadLink href={value.url} download={value.orig_name || value.path}>
+			<DownloadLink href={value.sources_file.url} download={value.sources_file.orig_name || value.sources_file.path}>
 				<IconButton Icon={Download} label={i18n("common.download")} />
 			</DownloadLink>
 		{/if}
@@ -63,6 +64,7 @@
 	<AudioPlayer
 		{value}
 		{label}
+		{root}
 		{i18n}
 		{waveform_settings}
 		{waveform_options}