mezzanine edits

extensions/src/doodlebot/CustomArgument.svelte

@@ -0,0 +1,83 @@
+<script lang="ts">
+  import Extension, { soundFiles } from ".";
+  import { ParameterOf, ArgumentEntry, ArgumentEntrySetter } from "$common";
+
+  /**
+   * Modify this type to match the argument you're developing this UI for.
+   * The first generic argumen is a reference to your extension.
+   * The second generic argumen is the name of the block function this argument belongs to.
+   * The third generic argumen is the index of the argument (i.e. is the functions 2nd argument? Then it's index would be 1)
+   */
+  type Value = ParameterOf<Extension, "playSoundFile", 0>;
+
+  // svelte-ignore unused-export-let
+  export let setter: ArgumentEntrySetter<Value>;
+
+  // svelte-ignore unused-export-let
+  export let current: ArgumentEntry<Value>;
+
+  // svelte-ignore unused-export-let
+  export let extension: Extension;
+  let value = current.value;
+  $: text = value;
+
+  $: setter({ value, text });
+</script>
+
+<div>
+  {#each soundFiles as f}
+    {#if value == f}
+      <div
+        class="goog-menuitem goog-option goog-option-selected"
+        role="menuitemcheckbox"
+        aria-checked="true"
+        id=":b"
+        style="user-select: none;"
+      >
+        <div class="goog-menuitem-content" style="user-select: none;">
+          <div class="goog-menuitem-checkbox" style="user-select: none;"></div>
+          <button
+            on:click={() => {
+              value = f;
+            }}
+          >
+            {f}
+          </button>
+        </div>
+      </div>
+    {/if}
+    {#if value != f}
+      <div
+        class="goog-menuitem goog-option goog-option-selected"
+        role="menuitemcheckbox"
+        aria-checked="true"
+        id=":b"
+        style="user-select: none;"
+      >
+        <div class="goog-menuitem-content" style="user-select: none;">
+          <button
+            on:click={() => {
+              value = f;
+            }}
+          >
+            {f}
+          </button>
+        </div>
+      </div>
+    {/if}
+  {/each}
+</div>
+
+<style>
+  button {
+    background-color: rgb(19, 236, 175);
+    border-color: rgb(11, 142, 105);
+    border: none;
+    font:
+      normal 13px "Helvetica Neue",
+      Helvetica,
+      sans-serif;
+    color: #000000;
+    font-weight: bold;
+  }
+</style>

extensions/src/doodlebot/Doodlebot.ts

@@ -147,6 +147,90 @@ export default class Doodlebot {
 
     private isStopped = true; // should this be initializeed more intelligently?
 
+    public newSounds: string[] = [];
+    public newImages: string[] = [];
+    private soundFiles = ['Scn2ALL.wav', 'Huh_sigh.wav', 'HMMMM.wav', 'Scn4bALL.wav', 'Sch1Whistle.wav', 'Scn4ALL.wav', '5.wav', 'Hmm.wav', '1.wav', 'Scn6Whistle.wav', 'Scn4Whistle.wav', 'Yay.wav', '3.wav', '4.wav', 'Scn6ALL.wav', '8.wav', 'mmMMmmm.wav', '9.wav', 'Scn2Whistle.wav', 'Scn6Voice.wav', '2.wav', 'NO.wav', '7.wav', 'emmemm.wav', 'Scn4bVoice.wav', 'Scn4Voice.wav', 'mumbleandhum.wav', 'Scn2Voice.wav', 'hello.wav', 'OK.wav', '6.wav', 'gotit.wav', 'Scn1ALL.wav', 'Scn1Voice.wav']
+    private imageFiles = [
+        "hannah.jpg",
+        "sad.png",
+        "13confused.png",
+        "newhannah.jpg",
+        "RGB24bits_320x240.png",
+        "1sleep.png",
+        "wink.png",
+        "panda.gif",
+        "sleep.png",
+        "angry.png",
+        "base_v2.png",
+        "[email protected]",
+        "[email protected]",
+        "annoyed.png",
+        "8confused.png",
+        "4asleep.png",
+        "13asleep.png",
+        "14asleep.png",
+        "colorcheck_320x240.png",
+        "3confused.png",
+        "4sleep.png",
+        "12asleep.png",
+        "2asleep.png",
+        "NTSCtest_320x240.png",
+        "happy.png",
+        "angry_RTeye_closed.bmp",
+        "15confused.png",
+        "asleep.png",
+        "angry_mouth.bmp",
+        "11asleep.png",
+        "6confused.png",
+        "9confused.png",
+        "disgust.png",
+        "angry_LTeye-closed.bmp",
+        "animesmileinvertedsmall.png",
+        "love.png",
+        "a.out",
+        "15asleep.png",
+        "14confused.png",
+        "db_animation-test.gif",
+        "5confused.png",
+        "PALtest_320x240.png",
+        "9asleep.png",
+        "surprise.png",
+        "5asleep.png",
+        "sadface.png",
+        "10asleep.png",
+        "3asleep.png",
+        "10confused.png",
+        "2confused.png",
+        "engaged.png",
+        "angry_mouth_closed.bmp",
+        "RGBParrot_320x240.png",
+        "page7orig.jpg",
+        "somethingWrong.png",
+        "confused.png",
+        "11confused.png",
+        "ball.gif",
+        "7confused.png",
+        "12confused.png",
+        "worried.png",
+        "animesmileinverted.png",
+        "angry_LTeye.bmp",
+        "7asleep.png",
+        "panda.jpg",
+        "animesmile.png",
+        "cambridge24bit_320x240.png",
+        "[email protected]",
+        "8asleep.png",
+        "6asleep.png",
+        "fear.png",
+        "1asleep.png",
+        "3sleep.png",
+        "angry_cheek.bmp",
+        "4confused.png",
+        "base_v1.png",
+        "2sleep.png",
+        "1confused.png",
+        "angry_RTeye.bmp"
+    ];
     private sensorData = ({
         bumper: { front: 0, back: 0 },
         altimeter: 0,
@@ -282,8 +366,14 @@ export default class Doodlebot {
 
     private async onWebsocketMessage(event: MessageEvent) {
         console.log("websocket message", { event });
-        const text = await event.data.text();
-        console.log(text);
+        try {
+            const text = await event.data.text();
+            console.log(text);
+        }
+        catch (e) {
+            console.log(JSON.stringify(event));
+            console.log("Error receiving message: ", e);
+        }
     }
 
     private invalidateWifiConnection() {
@@ -335,6 +425,29 @@ export default class Doodlebot {
         return this.sensorData[type];
     }
 
+    async findImageFiles() {
+        while (!this.connection) {
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+        let endpoint = "http://" + this.connection.ip + ":8080/images/"
+        console.log(endpoint);
+        let uploadedImages = await this.fetchAndExtractList(endpoint);
+        return uploadedImages.filter(item => !this.imageFiles.includes(item));
+    }
+
+    async findSoundFiles() {
+        while (!this.connection) {
+            await new Promise(resolve => setTimeout(resolve, 100));
+        }
+        if (!this.connection) return [];
+        let endpoint = "http://" + this.connection.ip + ":8080/sounds/"
+        let uploadedSounds = await this.fetchAndExtractList(endpoint);
+        console.log("uploaded");
+        console.log(uploadedSounds);
+        console.log(uploadedSounds.filter(item => !this.soundFiles.includes(item)));
+        return uploadedSounds.filter(item => !this.soundFiles.includes(item));
+    }
+
     /**
      * 
      * @param type 
@@ -440,7 +553,7 @@ export default class Doodlebot {
         }
     }
 
-    setIP(ip: string) {
+    async setIP(ip: string) {
         this.connection ??= { ip };
         this.saveIP(ip);
         return this.connection.ip = ip;
@@ -519,6 +632,103 @@ export default class Doodlebot {
         });
     }
 
+    parseWavHeader(uint8Array) {
+        const dataView = new DataView(uint8Array.buffer);
+
+        // Extract sample width, number of channels, and sample rate
+        const sampleWidth = dataView.getUint16(34, true) / 8; // Sample width in bytes (16-bit samples = 2 bytes, etc.)
+        const channels = dataView.getUint16(22, true); // Number of channels
+        const rate = dataView.getUint32(24, true); // Sample rate
+        const byteRate = dataView.getUint32(28, true); // Byte rate
+        const blockAlign = dataView.getUint16(32, true); // Block align
+        const dataSize = dataView.getUint32(40, true); // Size of the data chunk
+
+        const frameSize = blockAlign; // Size of each frame in bytes
+
+        return {
+            sampleWidth,
+            channels,
+            rate,
+            frameSize,
+            dataSize
+        };
+    }
+
+    splitIntoChunks(uint8Array, framesPerChunk) {
+        const headerInfo = this.parseWavHeader(uint8Array);
+        const { frameSize } = headerInfo;
+        const chunkSize = framesPerChunk * frameSize; // Number of bytes per chunk
+        const chunks = [];
+
+        // Skip the header (typically 44 bytes)
+        const dataStart = 44;
+
+        for (let i = dataStart; i < uint8Array.length; i += chunkSize) {
+            const chunk = uint8Array.slice(i, i + chunkSize);
+            chunks.push(chunk);
+        }
+
+        return chunks;
+    }
+
+
+    async sendAudioData(uint8Array: Uint8Array) {
+        let CHUNK_SIZE = 1024;
+        let ip = this.connection.ip;
+        const ws = makeWebsocket(ip, '8877');
+
+        ws.onopen = () => {
+            console.log('WebSocket connection opened');
+
+            let offset = 0;
+
+            let { sampleWidth, channels, rate } = this.parseWavHeader(uint8Array);
+            let first = "(1," + String(sampleWidth) + "," + String(channels) + "," + String(rate) + ")";
+            console.log(first);
+            ws.send(first);
+            let chunks = this.splitIntoChunks(uint8Array, CHUNK_SIZE);
+            let i = 0;
+            async function sendNextChunk() {
+                if (i >= chunks.length) {
+                    console.log('All data sent');
+                    ws.close();
+                    return;
+                }
+
+                // Calculate end position of the current chunk
+                //const end = Math.min(offset + CHUNK_SIZE, uint8Array.length);
+                const chunk = chunks[i];
+                console.log("sending");
+
+                const binaryString = Array.from(chunk).map((byte: any) => String.fromCharCode(byte)).join('');;
+                const base64Data = btoa(binaryString);
+                const jsonData = JSON.stringify({ audio_data: base64Data });
+                console.log(jsonData);
+                ws.send(jsonData);
+                i = i + 1;
+                sendNextChunk();
+            }
+
+            // Start sending chunks
+            sendNextChunk();
+        };
+
+        ws.onerror = (error) => {
+            console.error('WebSocket error:', error);
+        };
+
+        ws.onmessage = (event) => {
+            const response = JSON.parse(event.data);
+            if (response.type === 'ack' && response.status === 'received') {
+                console.log('Message successfully received by server');
+            }
+        }
+
+        ws.onclose = () => {
+            console.log('WebSocket connection closed');
+        };
+    }
+
     /**
      * 
      * @param credentials 
@@ -627,10 +837,46 @@ export default class Doodlebot {
         await this.sendWebsocketCommand(command.display, value);
     }
 
-    async displayText(text: string) {
+    // Function to fetch and parse HTML template
+    async fetchAndExtractList(endpoint) {
+        try {
+            // Fetch the HTML template from the endpoint
+            const response = await fetch(endpoint);
+            // if (!response.ok) {
+            //     throw new Error('Network response was not ok');
+            // }
+
+            // Get the HTML text
+            const htmlText = await response.text();
+
+            console.log(response);
+            console.log(htmlText);
+
+            // Parse the HTML text into a DOM structure
+            const parser = new DOMParser();
+            const doc = parser.parseFromString(htmlText, 'text/html');
+
+            // Extract all <li> elements
+            const listItems = doc.querySelectorAll('li');
+
+            // Get the text content of each <li> element
+            const itemNames = Array.from(listItems).map(li => li.textContent.trim());
+
+            return itemNames;
+        } catch (error) {
+            console.error('Error fetching or parsing HTML:', error);
+            return [];
+        }
+    }
+
+    async displayText(text: string, size: string) {
+        //(d,F,[number]) == s, m, or l
+        await this.sendWebsocketCommand(command.display, "F", size);
         await this.sendWebsocketCommand(command.display, "t", text);
     }
 
+
+
     /**
      * NOTE: Consider making private
      * @param command