This is the code for the first version of the Camera / Sender.
See here for the full write-up of the project at this stage.
#include <Arduino.h>
#include "esp_camera.h"
#include "FS.h"
#include "SD.h"
#include <rom/crc.h>
#include "JPEGDEC.h"
#include <Preferences.h>
// Pin configuration
#define PIN_AUDIO_LEFT 1 // D0 (GPIO 1)
#define PIN_AUDIO_RIGHT 2 // D1 (GPIO 2)
#define BUTTON_PIN 4 // D3 (GPIO 4)
#define SD_CS_AND_LED 21 // GPIO21 controls BOTH the LED and SD Chip Select
// Setup for the button
volatile bool buttonPressed = false;
volatile bool isTransmitting = false;
unsigned long lastDebounceTime = 0;
const unsigned long DEBOUNCE_DELAY = 250;
// Initialize the Preferences object for non-volatile storage
Preferences preferences;
// PWM/LEDC Settings
const int PWM_FREQ = 1000; // Initialization frequency (1000 Hz)
const int PWM_RES = 8; // 8-bit resolution (values 0 - 255)
const int TONE_DURATION_MS = 65; // Duration per tone in milliseconds
// 📐 SCREEN AND BUFFER SIZE
const int IMG_WIDTH = 400;
const int IMG_HEIGHT = 300;
const size_t EXPECTED_DATA_SIZE = (IMG_WIDTH * IMG_HEIGHT) / 2;
// --- CONFIGURABLE DOWNSAMPLING FACTOR ---
// Choose from:
// 0 : Keep original resolution (1200x1600)
// 2 : Reduce image size by 2x (600x800)
// 4 : Reduce image size by 4x (300x400)
// 8 : Reduce image size by 8x (150x200)
const int DOWNSAMPLE_FACTOR = 4;
size_t bytesSent = 0;
bool usingSD = false; // To verify if the user_LED can be used
bool ledState = false;
const int FREQUENCIES[16] = {
1000, 1200, 1400, 1600, 1800, 2000, 2200, 2400,
2600, 2800, 3000, 3200, 3400, 3600, 3800, 4000
};
const int PALETTE_SIZE = 6;
const uint8_t SPECTRA6_PALETTE[PALETTE_SIZE][3] = {
{205, 208, 200}, // 0: Realistic E-Paper White (Matte/Off-white)
{15, 15, 20}, // 1: Realistic E-Paper Black (Dark Charcoal)
{135, 28, 24}, // 2: Realistic E-Paper Red (Deep Earthy Red)
{215, 175, 40}, // 3: Realistic E-Paper Yellow (Mustard Yellow)
{55, 105, 60}, // 4: Realistic E-Paper Green (Olive Green)
{45, 85, 140} // 5: Realistic E-Paper Blue (Slate Blue)
};
struct ErrorRGB {
float r = 0;
float g = 0;
float b = 0;
};
// Instance of the JPEGDEC decoder
JPEGDEC jpeg;
// Pointers for the dithering process
ErrorRGB* currentErrorLine = NULL;
ErrorRGB* nextErrorLine = NULL;
int scaledWidth = 0;
uint8_t* output4BitBuffer = NULL;
size_t packedBufferIdx = 0;
uint8_t packedByte = 0;
bool isLeftNibble = true;
// =========================================================================
// 📸 Finds the closest color in the fixed palette using Euclidean distance
// =========================================================================
uint8_t findClosestPaletteIndex(int r, int g, int b) {
uint8_t closestIndex = 0;
long minDistance = INT_MAX;
for (int i = 0; i < PALETTE_SIZE; i++) {
long dr = r - SPECTRA6_PALETTE[i][0];
long dg = g - SPECTRA6_PALETTE[i][1];
long db = b - SPECTRA6_PALETTE[i][2];
long distance = (dr * dr) + (dg * dg) + (db * db);
if (distance < minDistance) {
minDistance = distance;
closestIndex = i;
}
}
return closestIndex;
}
// =========================================================================
// 📸 Callback function that processes and packs the image pixel by pixel
// =========================================================================
int JPEGDraw(JPEGDRAW *pDraw) {
int y = pDraw->y;
for (int j = 0; j < pDraw->iHeight; j++) {
for (int x = 0; x < pDraw->iWidth; x++) {
// Convert RGB565 pixel to 8-bit RGB components
uint16_t pixel565 = pDraw->pPixels[(j * pDraw->iWidth) + x];
int r = ((pixel565 >> 11) & 0x1F) << 3;
int g = ((pixel565 >> 5) & 0x3F) << 2;
int b = (pixel565 & 0x1F) << 3;
// Apply error diffusion from previous pixels
r += currentErrorLine[x].r;
g += currentErrorLine[x].g;
b += currentErrorLine[x].b;
r = constrain(r, 0, 255);
g = constrain(g, 0, 255);
b = constrain(b, 0, 255);
uint8_t colorIndex = findClosestPaletteIndex(r, g, b);
// --- 4-BIT PACKING STRATEGY (Live during dithering) ---
if (isLeftNibble) {
packedByte = (colorIndex << 4); // Store index in the high (left) nibble
isLeftNibble = false;
} else {
packedByte |= (colorIndex & 0x0F); // Add index to the low (right) nibble
output4BitBuffer[packedBufferIdx++] = packedByte; // Write byte to the output buffer
isLeftNibble = true;
}
// Floyd-Steinberg error distribution
float errR = r - SPECTRA6_PALETTE[colorIndex][0];
float errG = g - SPECTRA6_PALETTE[colorIndex][1];
float errB = b - SPECTRA6_PALETTE[colorIndex][2];
if (x + 1 < scaledWidth) {
currentErrorLine[x + 1].r += errR * (7.0 / 16.0);
currentErrorLine[x + 1].g += errG * (7.0 / 16.0);
currentErrorLine[x + 1].b += errB * (7.0 / 16.0);
}
if (x - 1 >= 0) {
nextErrorLine[x - 1].r += errR * (3.0 / 16.0);
nextErrorLine[x - 1].g += errG * (3.0 / 16.0);
nextErrorLine[x - 1].b += errB * (3.0 / 16.0);
}
nextErrorLine[x].r += errR * (5.0 / 16.0);
nextErrorLine[x].g += errG * (5.0 / 16.0);
nextErrorLine[x].b += errB * (5.0 / 16.0);
if (x + 1 < scaledWidth) {
nextErrorLine[x + 1].r += errR * (1.0 / 16.0);
nextErrorLine[x + 1].g += errG * (1.0 / 16.0);
nextErrorLine[x + 1].b += errB * (1.0 / 16.0);
}
}
// Copy error distribution to the current line and clear the next line buffer
memcpy(currentErrorLine, nextErrorLine, scaledWidth * sizeof(ErrorRGB));
memset(nextErrorLine, 0, scaledWidth * sizeof(ErrorRGB));
}
yield();
return 1;
}
// =========================================================================
// 📸 Dither the image to the 6 color ePaper colors
// =========================================================================
void processAndDitherImage(camera_fb_t *fb) {
if (!fb) return;
if (jpeg.openRAM(fb->buf, fb->len, JPEGDraw)) {
int originalWidth = jpeg.getWidth();
int originalHeight = jpeg.getHeight();
// Calculate scaled resolution
int divisor = (DOWNSAMPLE_FACTOR == 0) ? 1 : DOWNSAMPLE_FACTOR;
scaledWidth = originalWidth / divisor;
int scaledHeight = originalHeight / divisor;
// Calculate exact required byte size for the 4-bit output buffer
size_t totalPixels = scaledWidth * scaledHeight;
size_t total4BitBytes = (totalPixels + 1) / 2; // +1 to account for rounding errors on odd pixel counts
Serial.printf("🎨 [Image] Start Dithering. Output format: %dx%d (%d pixels).\n", scaledWidth, scaledHeight, totalPixels);
Serial.printf("🎨 [Image] Memory required for 4-bit buffer: %d bytes.\n", total4BitBytes);
// Buffer allocation
currentErrorLine = (ErrorRGB*)calloc(scaledWidth, sizeof(ErrorRGB));
nextErrorLine = (ErrorRGB*)calloc(scaledWidth, sizeof(ErrorRGB));
output4BitBuffer = (uint8_t*)malloc(total4BitBytes);
if (!currentErrorLine || !nextErrorLine || !output4BitBuffer) {
Serial.println("Memory allocation failed!");
if (currentErrorLine) free(currentErrorLine);
if (nextErrorLine) free(nextErrorLine);
if (output4BitBuffer) free(output4BitBuffer);
jpeg.close();
return;
}
// Reset packing variables prior to decoding
packedBufferIdx = 0;
packedByte = 0;
isLeftNibble = true;
// Set scaling configuration flags for the decoder
int scaleFlags = 0;
if (DOWNSAMPLE_FACTOR == 2) scaleFlags = JPEG_SCALE_HALF;
else if (DOWNSAMPLE_FACTOR == 4) scaleFlags = JPEG_SCALE_QUARTER;
else if (DOWNSAMPLE_FACTOR == 8) scaleFlags = JPEG_SCALE_EIGHTH;
// Start decoding and dithering process
jpeg.decode(0, 0, scaleFlags);
jpeg.close();
// If the final pixel count was odd, write the remaining last incomplete byte
if (!isLeftNibble) {
output4BitBuffer[packedBufferIdx++] = packedByte;
}
Serial.printf("🎨 [Image] Dithering and packing completed! Effectively written: %d bytes.\n", packedBufferIdx);
// Clean up temporary runtime memory allocations
free(currentErrorLine);
free(nextErrorLine);
//free(output4BitBuffer);
} else {
Serial.println("🎨 [Image] JPEGDEC could not open the camera buffer!");
}
}
// =========================================================================
// 🕹️ Handle the button press
// =========================================================================
void IRAM_ATTR handleButtonPress() {
unsigned long currentTime = millis();
// Respond ONLY if the debounce time has passed AND we are not already transmitting!
if (!isTransmitting && (currentTime - lastDebounceTime > DEBOUNCE_DELAY)) {
buttonPressed = true;
lastDebounceTime = currentTime;
}
}
// =========================================================================
// 🔊 Play a live stereo tone over D0/D1 for a specific duration
// =========================================================================
void playStereoTonePWM(int freqLeft, int freqRight, int durationMs) {
if (freqLeft == 0 && freqRight == 0) {
// Silence: set the frequency to 0
ledcWriteTone(PIN_AUDIO_LEFT, 0); // Mute Left
ledcWriteTone(PIN_AUDIO_RIGHT, 0); // Mute Right
} else {
// The hardware now directly generates a pure, stable square wave
ledcWriteTone(PIN_AUDIO_LEFT, freqLeft);
ledcWriteTone(PIN_AUDIO_RIGHT, freqRight);
}
// Wait exactly the configured tone duration (e.g., 10ms)
delay(durationMs);
}
// =========================================================================
// 🔊 Processes a live byte (2 pixels) into hardware PWM tones
// =========================================================================
void playDataBytePWM(uint8_t dataByte) {
uint8_t highNibble = (dataByte >> 4) & 0x0F; // Left Channel
uint8_t lowNibble = dataByte & 0x0F; // Right Channel
// Start a clean new row in the serial monitor interface every 40 bytes transmitted
if (bytesSent % 40 == 0) {
Serial.printf("\n➡️ [%06d] ", bytesSent);
}
bytesSent++;
// Print strictly the 2 hexadecimal characters of the byte (e.g., "01" or "50")
Serial.printf("%02X ", dataByte);
if (!usingSD) {
ledState = !ledState;
digitalWrite(SD_CS_AND_LED, ledState ? LOW : HIGH);
}
// Seamlessly output the complete byte in stereo (0% dead-time, constant energy output!)
playStereoTonePWM(FREQUENCIES[highNibble], FREQUENCIES[lowNibble], TONE_DURATION_MS);
}
// =========================================================================
// 📸 Ensure that the camera configuration is correctly loaded upon startup!
// =========================================================================
bool initCamera() {
camera_config_t config;
config.ledc_channel = LEDC_CHANNEL_0;
config.ledc_timer = LEDC_TIMER_0;
config.pin_d0 = 15;
config.pin_d1 = 17;
config.pin_d2 = 18;
config.pin_d3 = 16;
config.pin_d4 = 14;
config.pin_d5 = 12;
config.pin_d6 = 11;
config.pin_d7 = 48;
config.pin_xclk = 10;
config.pin_pclk = 13;
config.pin_vsync = 38;
config.pin_href = 47;
config.pin_sscb_sda = 40;
config.pin_sscb_scl = 39;
config.pin_pwdn = -1;
config.pin_reset = -1;
config.xclk_freq_hz = 20000000;
// We start the initialization with a lightweight JPEG frame to prevent malloc errors
config.frame_size = FRAMESIZE_UXGA; // 1600x1200
config.pixel_format = PIXFORMAT_JPEG;
config.grab_mode = CAMERA_GRAB_WHEN_EMPTY;
config.fb_location = CAMERA_FB_IN_PSRAM; // Requires Tools -> PSRAM -> OPI PSRAM!
config.jpeg_quality = 16;
config.fb_count = 2;
esp_err_t err = esp_camera_init(&config);
if (err != ESP_OK) {
Serial.printf("❌ [Camera] Init failed with error 0x%x\n", err);
return false;
}
// Let the camera registers stabilize (discard the first 4 volatile frames)
for(int i = 0; i < 4; i++) {
camera_fb_t * fb_warmup = esp_camera_fb_get();
if(fb_warmup) {
esp_camera_fb_return(fb_warmup);
delay(50);
}
}
return true;
}
// =========================================================================
// 💾 Make sure the SD Card and LED are not used at the same time
// =========================================================================
void toggleSD(bool status) {
if (!status) {
// Ensure no SD card operations are happening right now!
delay(100);
usingSD = false;
ledState = false;
digitalWrite(SD_CS_AND_LED, ledState ? LOW : HIGH);
}
if (status) {
// TURN THE LED OFF (Crucial before next SD card activity) ---
usingSD = true;
ledState = false;
digitalWrite(SD_CS_AND_LED, ledState ? LOW : HIGH);
delay(100);
}
}
// =========================================================================
// 💾 Save the buffer to the SD Card so we can review later
// =========================================================================
void saveBufferToSD(uint8_t* buffer, size_t len, const char* baseName, const char* extension) {
char filename[32];
// Open Preferences ("storage") in read/write mode (false)
// "file_storage" acts as the namespace
preferences.begin("file_storage", false);
// Retrieve the current counter value. If the key "counter" does not exist,
// it will return the default value (0 in this case).
int currentCounter = preferences.getInt("counter", 0);
// Increment the counter for the new file entry
currentCounter++;
// Generate a unique filename string, e.g., /photo_raw_1.jpg
sprintf(filename, "/%s_%d.%s", baseName, currentCounter, extension);
Serial.printf("💾 [SD Card] Saving to SD: %s (%d bytes)...\n", filename, len);
// The actual writing to the SD Card
toggleSD(true); // make sure the LED pin cannot interrupt
File file = SD.open(filename, FILE_WRITE);
if(!file){
Serial.println("❌ [SD Card] Error opening file for writing!");
return;
}
size_t written = file.write(buffer, len);
if(written == len){
Serial.println("✅ [SD Card] Successfully saved!");
} else {
Serial.printf("❌ [SD Card] Error: Only %d of %d bytes written.\n", written, len);
}
file.close();
toggleSD(false); // enable the LED pin after file was saved
// Store the NEW index counter value in non-volatile memory for the next boot cycle
preferences.putInt("counter", currentCounter);
// Close the Preferences session cleanly
preferences.end();
}
// =========================================================================
// 📸 After clicking the button, this is used to take the photo and send it
// =========================================================================
void processAndTransmitPhoto() {
isTransmitting = true;
bytesSent = 0;
Serial.println("🕹️ [BUTTON] Button pressed!");
Serial.println("📸 [CAMERA] Capturing high-res UXGA (1200x1600) frame...");
camera_fb_t * fb = esp_camera_fb_get();
if (!fb) {
Serial.println("❌ [CAMERA] Failed to capture a high-res frame!");
isTransmitting = false; buttonPressed = false; return;
}
Serial.printf("✅ [CAMERA] Frame captured (%dx%d). Starting downsampling and dithering...\n", fb->width, fb->height);
// Allocate exactly 240,000 bytes inside PSRAM for the audio modem data stream buffer
uint8_t* compressedBuffer = (uint8_t*)ps_malloc(EXPECTED_DATA_SIZE);
if (!compressedBuffer) {
Serial.println("❌ [PSRAM] No space available for the 240KB transmission buffer!");
esp_camera_fb_return(fb);
isTransmitting = false;
buttonPressed = false;
return;
}
// Converts the raw 1200x1600 RGB frame into the compressed 240 KB dithered audio buffer
processAndDitherImage(fb);
// Release the camera frame buffer memory
esp_camera_fb_return(fb);
// Save the original raw JPEG file arriving straight out of the camera sensor
saveBufferToSD(fb->buf, fb->len, "foto_raw", "jpg");
// And store the newly generated dithered photo:
saveBufferToSD(output4BitBuffer, packedBufferIdx, "dither_4bit", "bin");
// Populate the buffer allocation tracking matrix with a distinct color sequence test pattern
// This circumvents both camera captures and downsampling executions fully for this text test run!
//for (size_t i = 0; i < EXPECTED_DATA_SIZE; i++) {
// We package two distinct palette indexes (0 to 5) consecutively per container byte
// uint8_t pixel1 = (i) % PALETTE_SIZE; // High Nibble (Left)
// uint8_t pixel2 = (i + 1) % PALETTE_SIZE; // Low Nibble (Right)
// compressedBuffer[i] = (pixel1 << 4) | (pixel2 & 0x0F);
//}
// Dispatch the solid 2-second initial pilot tone (800 Hz)
Serial.println("🔊 [START] Sending start tone...");
playStereoTonePWM(800, 800, 2000);
// Dispatch 3 synchronization header framing bytes
Serial.println("🔊 [SYNC] Sending 3 sync bytes...");
Serial.print("➡️ [SYNC ] ");
bytesSent = 1; // set to 1, to avoid it being printed before SYNC
playDataBytePWM(0x1B);
playDataBytePWM(0x53);
playDataBytePWM(0x46);
// This can be used for receiver-side calibration procedures
//for (int k = 0; k < 16; k++) {
// playStereoTonePWM(FREQUENCIES[k], FREQUENCIES[k], TONE_DURATION_MS);
//}
// Stream out the packed bytes over true hardware stereo channels (100 ms duration per byte allocation block)
Serial.print("\n🔊 [DATA] Streaming initiated...");
bytesSent = 0;
for (size_t i = 0; i < packedBufferIdx; i++) {
playDataBytePWM(output4BitBuffer[i]);
}
// Compute the precise CRC32 checksum across the processed bytes dataset array blocks
uint32_t checksum = crc32_le(0, output4BitBuffer, packedBufferIdx);
Serial.printf("\n🔊 [CRC] Data payload complete. Computed CRC32 checksum: 0x%08X\n", checksum);
Serial.print("➡️ [CRC ] ");
bytesSent = 1; // set to 1, to avoid it being printed before CRC
// Broadcast the 4 calculated Checksum signature payload bytes via audio outputs (each 100 ms block)
for (int i = 0; i < 4; i++) {
uint8_t crcByte = (checksum >> (i * 8)) & 0xFF;
playDataBytePWM(crcByte);
}
// Output final closing termination audio burst signal sequence (500 ms at 4500Hz)
Serial.println("\n🔊 [FINISH] Sending finish sign-off burst tone...");
playStereoTonePWM(4500, 4500, 500);
Serial.println("🎉 [DONE!] Audio-based data transmission sequence completed successfully!");
// Mute system and scrub local buffers
ledcWriteTone(PIN_AUDIO_LEFT, 0);
ledcWriteTone(PIN_AUDIO_RIGHT, 0);
if (!usingSD) {
ledState = false;
digitalWrite(SD_CS_AND_LED, ledState ? LOW : HIGH);
}
free(compressedBuffer);
delay(1000);
buttonPressed = false;
isTransmitting = false;
}
// =========================================================================
// 🚀 Initialization (SETUP)
// =========================================================================
void setup() {
Serial.begin(115200);
delay(1000);
Serial.println("\nSystem booting...");
// Initialize PSRAM interface and verify state allocations
if (psramInit()) {
Serial.println("✅ [PSRAM] Initialized successfully.");
} else {
Serial.println("❌ [PSRAM] Initialization failed! Enable OPI PSRAM configuration under Tools menu.");
while(1);
}
// Initialize the camera with the correct pins and resolution
if(!initCamera()) {
Serial.println("❌ [Camera] Install the camera module correctly on the XIAO Sense.");
} else {
Serial.println("✅ [Camera] Loaded correctly.");
}
// Initialize SD Card
if (!SD.begin(SD_CS_AND_LED)) {
Serial.println("❌ [SD Card] Initialization failed!");
} else {
Serial.println("✅ [SD Card] Ready");
}
uint8_t cardType = SD.cardType();
if(cardType == CARD_NONE){
Serial.println("❌ [SD Card] No SD card inserted");
} else {
Serial.print("✅ [SD Card] Card Type: ");
if(cardType == CARD_MMC){
Serial.print("MMC");
} else if(cardType == CARD_SD){
Serial.print("SDSC");
} else if(cardType == CARD_SDHC){
Serial.print("SDHC");
} else {
Serial.print("UNKNOWN");
}
uint64_t cardSize = SD.cardSize() / (1024 * 1024);
Serial.printf(", Size: %lluMB\n", cardSize);
}
toggleSD(false);
// By enforcing a 1 Hz starting delta offset constraint initialization, the ESP32-S3
// ensures pin channels split across separate independent hardware timing generation modules!
ledcAttach(PIN_AUDIO_LEFT, PWM_FREQ, PWM_RES); // D0 -> Maps cleanly onto Hardware Timer A
ledcAttach(PIN_AUDIO_RIGHT, PWM_FREQ+1, PWM_RES); // D1 -> Maps cleanly onto Hardware Timer B
// Assign the pin and interrupt to the button
pinMode(BUTTON_PIN, INPUT_PULLUP);
attachInterrupt(digitalPinToInterrupt(BUTTON_PIN), handleButtonPress, FALLING);
// All setup is done!
Serial.println("🚀 [READY] Camera Online. Press button to take and send a photo!");
}
// =========================================================================
// 🔄 Main program - only looks for button press toggle (LOOP)
// =========================================================================
void loop() {
if (buttonPressed) {
buttonPressed = false;
processAndTransmitPhoto();
}
delay(10);
}